CSCfi · EarthModule · Jan 27, 2021 · Nov 9, 2020 · Nov 9, 2020 · Nov 9, 2020
diff --git a/.flake8 b/.flake8
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
diff --git a/.gitignore b/.gitignore
@@ -13,3 +13,8 @@
 ubuntu-xenial-16.04-cloudimg-console.log
 .ropeproject/
 .venv_metax/
+.env
+.env.*
+!.env.template
+*.pem
+/src/log
diff --git a/.travis.yml b/.travis.yml
@@ -3,39 +3,63 @@ dist: trusty
 language: python
 python: 3.6
 
-branches:
-  only:
-  - test
-  - stable
-
 cache: pip
 
 addons:
+  apt:
+    packages:
+      - xqilla
+      - libxqilla-dev
   postgresql: '9.6'
 
 services:
-- postgresql
+  - postgresql
+  - redis-server
 
-before_install:
-- openssl aes-256-cbc -K $encrypted_62ed3fb8af4c_key -iv $encrypted_62ed3fb8af4c_iv -in deploy-key.enc -out deploy-key -d
-- rm deploy-key.enc
-- chmod 600 deploy-key
-- mv deploy-key ~/.ssh/id_rsa
-- eval "$(ssh-agent -s)"
-- ssh-add ~/.ssh/id_rsa
-- sudo apt-get install xqilla libxqilla-dev
+env:
+  global:
+    - AUTH_SERVER_LOGOUT_URL=http://logout
+    - DATACITE_PREFIX=10.12345
+    - DATACITE_URL=http://mockurl.com
+    - DATACITE_ETSIN_URL_TEMPLATE=http://mockurl.com/%s
+    - DATACITE_PASSWORD=not_set
+    - DATACITE_USERNAME=not_set
+    - DEBUG=false
+    - DJANGO_ENV=travis
+    - DJANGO_SECRET_KEY=lxpRfm8.JOUa7K.eOxZGtL*o+.zt.ybj
+    - DRAFT_ENABLED=true
+    - ELASTIC_SEARCH_HOSTS=https://metax.fd-test.csc.fi/es
+    - ELASTIC_SEARCH_USE_SSL=true
+    - METAX_DATABASE=metax_db_test
+    - METAX_DATABASE_PASSWORD=
+    - METAX_DATABASE_USER=metax_test
+    - OAI_BASE_URL=http://mockurl.com
+    - OAI_ETSIN_URL_TEMPLATE=http://mockurl.com/%s
+    - [email protected]
+    - REMS_ENABLED=true
+    - REMS_API_KEY=not_set
+    - REMS_BASE_URL=http://mockurl.com
+    - REMS_ETSIN_URL_TEMPLATE=http://mockurl.com/%s
+    - REMS_METAX_USER=not_set
+    - REMS_REPORTER_USER=not_set
+    - REMS_AUTO_APPROVER=not_set
+    - REMS_FORM_ID=99
+    - ENABLE_V1_ENDPOINTS=true
+    - ENABLE_V2_ENDPOINTS=true
 
 install:
 - pip install -r requirements.txt
-- sudo service postgresql restart
 
 before_script:
 - psql -U postgres < resources/sql/init_test.sql
-- sudo mkdir /var/log/metax-api
-- sudo chown -R $USER:$USER /var/log/metax-api
+- sudo mkdir -p src/log
+- sudo chown -R $USER:$USER src/log
 
 script:
-- cd src && flake8 . && python manage.py migrate metax_api && coverage run --source="." manage.py test metax_api && COVERALLS_REPO_TOKEN=$coveralls_token coveralls && $TRAVIS_BUILD_DIR/.travis-deploy.sh
+- cd src && flake8 .
+- python manage.py migrate metax_api
+- coverage run --source="." manage.py test metax_api
+- COVERALLS_REPO_TOKEN=$coveralls_token coveralls
 
 notifications:
   flowdock:

diff --git a/AUTHORS b/AUTHORS
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,33 @@
+FROM python:3.6
+
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE 1
+
+WORKDIR /code
+
+COPY requirements.txt /code/
+
+RUN apt-get update && apt install xqilla libxerces-c-dev build-essential libssl-dev libffi-dev python-dev libxqilla-dev -y
+
+RUN pip install --upgrade pip wheel
+RUN pip install -r requirements.txt
+
+EXPOSE 8008
+EXPOSE 8006
+
+ARG METAX_DATABASE_HOST
+ARG REDIS_HOST
+ARG RABBITMQ_HOST
+ARG RABBIT_MQ_PASSWORD=guest
+ARG RABBIT_MQ_USER=guest
+ARG ELASTIC_SEARCH_HOST
+
+ENV METAX_DATABASE_HOST $METAX_DATABASE_HOST
+ENV REDIS_HOST $REDIS_HOST
+ENV RABBIT_MQ_HOSTS $RABBITMQ_HOST
+ENV RABBIT_MQ_PASSWORD $RABBIT_MQ_PASSWORD
+ENV RABBIT_MQ_USER $RABBIT_MQ_USER
+ENV ELASTIC_SEARCH_HOSTS $ELASTIC_SEARCH_HOST
+
+# CMD ["python", "/code/manage.py", "runserver", "0.0.0.0:8008"]
+CMD ["python", "manage.py", "runsslserver", "--certificate", ".certs/cert.pem","--key", ".certs/key.pem", "0.0.0.0:8008"]
diff --git a/ENV_VARS.md b/ENV_VARS.md
@@ -0,0 +1,62 @@
+# Required environmental variables
+
+copy .env.template to .env and fill the required values from below table. Required column tells if you have to have the variable in the .env file
+
+| Name                           | Required | Default                                                                               | Description                                                                                                |
+| ------------------------------ | -------- | ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- |
+| ADDITIONAL_USER_PROJECTS_PATH  | unknown  | ""                                                                                    | No clue if this is important                                                                               |
+| ALLOWED_HOSTS                  | no       |                                                                                       | defines which IP-addresses are allowed to access metax, DJANGO_ENV=local overrides this                    |
+| AUTH_SERVER_LOGOUT_URL         | unknown  |                                                                                       | Requires testing if this is needed                                                                         |
+| DATACITE_ETSIN_URL_TEMPLATE    | yes      |                                                                                       |
+| DATACITE_PASSWORD              | yes      |                                                                                       |
+| DATACITE_PREFIX                | yes      |                                                                                       |
+| DATACITE_URL                   | yes      |                                                                                       |
+| DATACITE_USERNAME              | yes      |                                                                                       |
+| DEBUG                          | no       | False                                                                                 |
+| DJANGO_ENV                     | no       | local                                                                                 | Specifies the environment, corresponds with the environments found in src/metax_api/settings/environments/ |
+| DJANGO_SECRET_KEY              | yes      |                                                                                       |
+| DRAFT_ENABLED                  | no       |                                                                                       |
+| ELASTIC_SEARCH_HOSTS           | yes      |                                                                                       | Elastic Search instance IP and port                                                                        |
+| ELASTIC_SEARCH_PORT            | no       | 9200                                                                                  | Is not used currently, but should be in the future                                                         |
+| ELASTIC_SEARCH_USE_SSL         | yes      |                                                                                       | Should Elastic Search queries use https                                                                    |
+| ERROR_FILES_PATH               | no       | src/log/errors                                                                        | Error file folder                                                                                          |
+| ES_CONFIG_DIR                  | no       | src/metax_api/tasks/refdata/refdata_indexer/resources/es-config                       | metax-ops compatibility                                                                                    |
+| LOCAL_REF_DATA_FOLDER          | no       | src/metax_api/tasks/refdata/refdata_indexer/resources/local-refdata                   | metax-ops compatibility                                                                                    |
+| LOGGING_DEBUG_HANDLER_FILE     | no       | src/log/metax_api.log                                                                 | metax-ops compatibility                                                                                    |
+| LOGGING_GENERAL_HANDLER_FILE   | no       | src/log/metax_api.log                                                                 | metax-ops compatibility                                                                                    |
+| LOGGING_JSON_FILE_HANDLER_FILE | no       | src/log/metax_api.json.log                                                            | metax-ops compatibility                                                                                    |
+| METAX_API_ROOT                 | yes      |                                                                                       | Metax internal variable, must be https                                                                     |
+| METAX_DATABASE                 | yes      |                                                                                       | Postgres database name                                                                                     |
+| METAX_DATABASE_PASSWORD        | yes      |                                                                                       | Postgres database password                                                                                 |
+| METAX_DATABASE_PORT            | no       | 5432                                                                                  | Postgres instance exposed port                                                                             |
+| METAX_DATABASE_USER            | yes      |                                                                                       | Postgres user which owns the database                                                                      |
+| METAX_ENV                      | no       | local_development                                                                     | Used by Metax internally, should be replaced with DJANGO_ENV in the future                                 |
+| OAI_BASE_URL                   | yes      |                                                                                       |
+| ORG_FILE_PATH                  | yes      | src/metax_api/tasks/refdata/refdata_indexer/resources/organizations/organizations.csv | metax-ops compatibility                                                                                    |
+| RABBIT_MQ_HOSTS                | yes      |                                                                                       | RabbitMQ instance IP                                                                                       |
+| RABBIT_MQ_PASSWORD             | no       | guest                                                                                 |
+| RABBIT_MQ_PORT                 | no       | 5672                                                                                  |
+| RABBIT_MQ_TTV_ENABLED          | no       |                                                                                       |
+| RABBIT_MQ_USER                 | no       | guest                                                                                 |
+| RABBIT_MQ_VHOST                | no       |                                                                                       |
+| RABBIT_MQ_VHOST_TTV            | no       |                                                                                       |
+| REDIS_HOST                     | yes      |                                                                                       | Redis instance IP                                                                                          |
+| REDIS_LOCALHOST_PORT           | unknown  | 6379                                                                                  | Not sure if all references to this are gone                                                                |
+| REDIS_PASSWORD                 | no       |                                                                                       |
+| REDIS_PORT                     | no       | 6379                                                                                  |
+| REDIS_TEST_DB                  | yes      |                                                                                       | Pick a number, any number                                                                                  |
+| REDIS_USE_PASSWORD             | no       | false                                                                                 |
+| REMS_API_KEY                   | no       |
+| REMS_AUTO_APPROVER             | no       |
+| REMS_BASE_URL                  | no       |
+| REMS_ENABLED                   | no       |
+| REMS_ETSIN_URL_TEMPLATE        | yes      |                                                                                       |
+| REMS_FORM_ID                   | yes      |                                                                                       |
+| REMS_METAX_USER                | no       |
+| REMS_REPORTER_USER             | no       |
+| SERVER_DOMAIN_NAME             | no       |
+| TRAVIS                         | no       | false                                                                                 | Metax internal used to figure out if Metax is running on Travis virtual machine                            |
+| V1_ENABLED                     | no       |                                                                                       |
+| V2_ENABLED                     | no       |                                                                                       |
+| VALIDATE_TOKEN_URL             | yes      |                                                                                       |
+| WKT_FILENAME                   | no       | src/metax_api/tasks/refdata/refdata_indexer/resources/uri_to_wkt.json                 |
diff --git a/LICENSE b/LICENSE
diff --git a/README.md b/README.md
@@ -13,3 +13,126 @@ License
 Copyright (c) 2018-2020 Ministry of Education and Culture, Finland
 
 Licensed under [GNU GPLv2 License](LICENSE)
+
+
+## Setting up local development environment
+
+### Prerequisites
+
+#### Docker-Engine
+
+Install Docker-Engine either following instructions below or looking up your platform specific instructions [from docs.docker.com][1] 
+
+##### Linux
+
+`$ curl -fsSL https://get.docker.com -o get-docker.sh`
+
+`$ sudo sh get-docker.sh`
+
+`$ sudo usermod -aG docker $USER`
+
+Log out and back in to activate non-sudo docker capabilities
+
+##### Mac
+
+https://docs.docker.com/docker-for-mac/install/
+
+#### Portainer (Optional)
+
+We will use portainer container management tool for monitoring various development dependencies. Command below will start portainer on every system startup.
+
+`$ docker volume create portainer_data` (optional for mac)
+
+`$ docker run -d -p 8000:8000 -p 9000:9000 --name=portainer --restart=always -v /var/run/docker.sock:/var/run/docker.sock -v portainer_data:/data portainer/portainer-ce`
+
+Finish the Portainer setup by logging in at http://localhost:9000, create a local endpoint from the Portainer interface. 
+
+#### Docker commands
+
+__NOTICE If you want to start the services everytime your computer boots, replace `--restart=unless-stopped` with `--restart=always`__
+
+Run the following docker commands to start services:
+
+##### Redis
+
+`docker run -d -p 6379:6379 --name metax-redis -v metax-redis:/data --restart=unless-stopped redis`
+
+##### Postgres
+
+`docker run -d -p 5432:5432 --name metax-postgres -v metax-postgres:/var/lib/postgresql96/data -e POSTGRES_USER=metax_db_user -e POSTGRES_PASSWORD=YMDLekQMqrVKcs3 -e POSTGRES_DB=metax_db --restart=unless-stopped  postgres:9`
+
+__NOTICE: copy values of `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_DB` into your `.env` files as `METAX_DATABASE_USER`, `METAX_DATABASE_PASSWORD` and `METAX_DATABASE`__
+
+##### Elasticsearch
+
+`docker run -d -p 9200:9200 -p 9300:9300 -v metax-es:/usr/share/elasticsearch/data --name metax-es -e discovery.type=single-node --restart=unless-stopped elasticsearch:7.9.2`
+
+##### RabbitMQ
+
+`docker run -d -p 5671:5671 -p 5672:5672 -p 15672:15672 -v metax-rabbitmq:/var/lib/rabbitmq --name metax-rabbitmq --restart=unless-stopped rabbitmq:3-management`
+
+#### mkcerts
+
+Install [mkcerts][2] and run `mkcert -install` and after it the following command:
+`mkcert -cert-file cert.pem -key-file key.pem 0.0.0.0 localhost 127.0.0.1 ::1 metax.csc.local 20.20.20.20`
+Move the `cert.pem` and `key.pem` to `src/.certs` folder (create the folder if not present).
+
+### Required environmental variables
+
+copy `src/metax_api/settings/.env.template` as `src/metax_api/settings/.env` and fill required variables, you can find examples in ENV_VARS.md
+
+### Create log directory 
+
+`mkdir -p src/log/errors`
+
+### Run Metax inside a container (Optional)
+
+Check the IP addresses of Redis, RabbitMQ, ElasticSearch and Postgres:9 either from Portainer container list (click the link in the container name to see all attributes) or by going to portainer network tab or by typing `docker container ps` followed by `docker network inspect bridge`
+
+Build new docker image from repository root with this command (change ip-addresses to real ones:
+
+`docker build -t metax-api:latest --build-arg METAX_DATABASE_HOST=xxx.xx.x.x --build-arg REDIS_HOST=xxx.xx.x.x --build-arg RABBITMQ_HOST=xxx.xx.x.x --build-arg ELASTIC_SEARCH_HOST=xxx.xx.x.x:xxxx .`
+
+Run the built container with command:
+
+`docker run -it --name metax-web --mount type=bind,source="$(pwd)"/src,target=/code -p 8008:8008 --rm metax-api:latest`
+
+You should see metax-server starting at port 8008 with hot reload enabled
+
+### Initial setup commands
+docker network inspect bridge
+IF you configured metax-container, access the command line of the container with `docker exec -it metax-web bash`
+
+__NOTICE: Skip activating virtualenv and navigating to src folder if you have metax running on container__
+
+Activate your python 3.6 virtualenv, `cd` into `src` folder and run following commands:
+
+setup the database with migrate command:
+
+`python manage.py migrate`
+
+__NOTICE: Skip following steps if your running metax on container and have terminal open in the container__
+
+start the development server with:
+`python manage.py runsslserver --certificate .certs/cert.pem --key .certs/key.pem 8008`
+
+Open another terminal and `cd` into `src`, and load the initial data with following commands: 
+
+__These commands must be run in both setups (container/not-container metax)__
+
+`python manage.py index_refdata`
+
+`python manage.py reload_refdata_cache`
+
+`python manage.py loadinitialdata`
+
+`python manage.py loaddata metax_api/tests/testdata/test_data.json` 
+
+run the tests with command `DJANGO_ENV=test python manage.py test --failfast --keepdb -v 0`
+
+Metax api is available from your browser at https://localhost:8008
+
+
+[1]: https://docs.docker.com/engine/install/
+[2]: https://github.com/FiloSottile/mkcert
+