Merge branch 'test' into Modify-JYU-catalog-logo

.gitignore

@@ -13,3 +13,6 @@
 ubuntu-xenial-16.04-cloudimg-console.log
 .ropeproject/
 .venv_metax/
+.env
+*.pem
+/src/log

.travis.yml

@@ -16,6 +16,9 @@ addons:
 services:
 - postgresql
 
+env:
+  - DJANGO_ENV=travis
+
 before_install:
 - openssl aes-256-cbc -K $encrypted_62ed3fb8af4c_key -iv $encrypted_62ed3fb8af4c_iv -in deploy-key.enc -out deploy-key -d
 - rm deploy-key.enc
@@ -35,7 +38,7 @@ before_script:
 - sudo chown -R $USER:$USER /var/log/metax-api
 
 script:
-- cd src && flake8 . && python manage.py migrate metax_api && coverage run --source="." manage.py test metax_api && COVERALLS_REPO_TOKEN=$coveralls_token coveralls && $TRAVIS_BUILD_DIR/.travis-deploy.sh
+- cd src && flake8 . && python manage.py migrate metax_api && coverage run --source="." && python manage.py test metax_api && COVERALLS_REPO_TOKEN=$coveralls_token coveralls && $TRAVIS_BUILD_DIR/.travis-deploy.sh
 
 notifications:
   flowdock:

Dockerfile

@@ -0,0 +1,33 @@
+FROM python:3.6
+
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE 1
+
+WORKDIR /code
+
+COPY requirements.txt /code/
+
+RUN apt-get update && apt install xqilla libxerces-c-dev build-essential libssl-dev libffi-dev python-dev libxqilla-dev -y
+
+RUN pip install --upgrade pip wheel
+RUN pip install -r requirements.txt
+
+EXPOSE 8008
+EXPOSE 8006
+
+ARG METAX_DATABASE_HOST
+ARG REDIS_HOST
+ARG RABBITMQ_HOST
+ARG RABBIT_MQ_PASSWORD=guest
+ARG RABBIT_MQ_USER=guest
+ARG ELASTIC_SEARCH_HOST
+
+ENV METAX_DATABASE_HOST $METAX_DATABASE_HOST
+ENV REDIS_HOST $REDIS_HOST
+ENV RABBIT_MQ_HOSTS $RABBITMQ_HOST
+ENV RABBIT_MQ_PASSWORD $RABBIT_MQ_PASSWORD
+ENV RABBIT_MQ_USER $RABBIT_MQ_USER
+ENV ELASTIC_SEARCH_HOSTS $ELASTIC_SEARCH_HOST
+
+# CMD ["python", "/code/manage.py", "runserver", "0.0.0.0:8008"]
+CMD ["python", "manage.py", "runsslserver", "--certificate", ".certs/cert.pem","--key", ".certs/key.pem", "0.0.0.0:8008"]

ENV_VARS.md

@@ -0,0 +1,33 @@
+# Required environmental variables
+
+copy .env.template to .env and fill following variables manually
+
+| Name                           | Description                              | example                                                                                                       |
+| ------------------------------ | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------- |
+| DATACITE_PREFIX                  |                                          |                                                                                                               |
+| DATACITE_URL                     |                                          |                                                                                                               |
+| DATACITE_ETSIN_URL_TEMPLATE    |                                          |                                                                                                               |
+| DATACITE_USERNAME              |                                          |                                                                                                               |
+| DATACITE_PASSWORD              |                                          |                                                                                                               |
+| DJANGO_SECRET_KEY              | replace with proper django secret key    |                                                                                                               |
+| ERROR_FILES_PATH               |                                          | /home/user/repo-root/logs/errors                                                                              |
+| ES_CONFIG_DIR                  |                                          | /home/user/repo-root/src/metax_api/tasks/refdata/refdata_indexer/resources/es-config/                         |
+| LOCAL_REF_DATA_FOLDER          |                                          | /home/user/repo-root/src/metax/metax-api/src/metax_api/tasks/refdata/refdata_indexer/resources/local-refdata/ |
+| LOGGING_DEBUG_HANDLER_FILE     |                                          | /home/user/repo-root/src/metax/metax-api/logs/metax_api.log                                                   |
+| LOGGING_GENERAL_HANDLER_FILE   |                                          | /home/user/repo-root/src/metax/metax-api/logs/metax_api.log                                                   |
+| LOGGING_JSON_FILE_HANDLER_FILE |                                          | /home/user/repo-root/src/metax/metax-api/logs/metax_api.json.log                                              |
+| METAX_DATABASE                 | postgres database, must be created first | metax                                                                                                         |
+| METAX_DATABASE_PASSWORD        | postgres owner of the database           |                                                                                                               |
+| METAX_DATABASE_PORT            |                                          | 5432                                                                                                          |
+| METAX_DATABASE_USER            |                                          | username                                                                                                      |
+| OAI_BASE_URL                   |                                          |                                                                                                               |
+| ORG_FILE_PATH                  |                                          | /home/user/repo-root/src/metax_api/tasks/refdata/refdata_indexer/resources/organizations/organizations.csv    |
+| RABBIT_MQ_PASSWORD             |                                          | guest                                                                                                         |
+| RABBIT_MQ_USER                 |                                          | guest                                                                                                         |
+| REDIS_LOCALHOST_PORT           |                                          | 6379                                                                                                          |
+| REDIS_PASSWORD                 |                                          |                                                                                                               |
+| REMS_ETSIN_URL_TEMPLATE        |                                          |                                                                                                               |
+| REMS_FORM_ID                   |                                          |                                                                                                               |
+| VALIDATE_TOKEN_URL             |                                          |                                                                                                               |
+| WKT_FILENAME                   |                                          | /home/user/repo-root/src/metax_api/tasks/refdata/refdata_indexer/resources/uri_to_wkt.json                    |
+

README.md

@@ -13,3 +13,126 @@ License
 Copyright (c) 2018-2020 Ministry of Education and Culture, Finland
 
 Licensed under [GNU GPLv2 License](LICENSE)
+
+
+## Setting up local development environment
+
+### Prerequisites
+
+#### Docker-Engine
+
+Install Docker-Engine either following instructions below or looking up your platform specific instructions [from docs.docker.com][1] 
+
+##### Linux
+
+`$ curl -fsSL https://get.docker.com -o get-docker.sh`
+
+`$ sudo sh get-docker.sh`
+
+`$ sudo usermod -aG docker $USER`
+
+Log out and back in to activate non-sudo docker capabilities
+
+##### Mac
+
+https://docs.docker.com/docker-for-mac/install/
+
+#### Portainer (Optional)
+
+We will use portainer container management tool for monitoring various development dependencies. Command below will start portainer on every system startup.
+
+`$ docker volume create portainer_data` (optional for mac)
+
+`$ docker run -d -p 8000:8000 -p 9000:9000 --name=portainer --restart=always -v /var/run/docker.sock:/var/run/docker.sock -v portainer_data:/data portainer/portainer-ce`
+
+Finish the Portainer setup by logging in at http://localhost:9000, create a local endpoint from the Portainer interface. 
+
+#### Docker commands
+
+__NOTICE If you want to start the services everytime your computer boots, replace `--restart=unless-stopped` with `--restart=always`__
+
+Run the following docker commands to start services:
+
+##### Redis
+
+`docker run -d -p 6379:6379 --name metax-redis -v metax-redis:/data --restart=unless-stopped redis`
+
+##### Postgres
+
+`docker run -d -p 5432:5432 --name metax-postgres -v metax-postgres:/var/lib/postgresql96/data -e POSTGRES_USER=metax_db_user -e POSTGRES_PASSWORD=YMDLekQMqrVKcs3 -e POSTGRES_DB=metax_db --restart=unless-stopped  postgres:9`
+
+__NOTICE: copy values of `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_DB` into your `.env` files as `METAX_DATABASE_USER`, `METAX_DATABASE_PASSWORD` and `METAX_DATABASE`__
+
+##### Elasticsearch
+
+`docker run -d -p 9200:9200 -p 9300:9300 -v metax-es:/usr/share/elasticsearch/data --name metax-es -e discovery.type=single-node --restart=unless-stopped elasticsearch:7.9.2`
+
+##### RabbitMQ
+
+`docker run -d -p 5671:5671 -p 5672:5672 -p 15672:15672 -v metax-rabbitmq:/var/lib/rabbitmq --name metax-rabbitmq --restart=unless-stopped rabbitmq:3-management`
+
+#### mkcerts
+
+Install [mkcerts][2] and run `mkcert -install` and after it the following command:
+`mkcert -cert-file cert.pem -key-file key.pem 0.0.0.0 localhost 127.0.0.1 ::1 metax.csc.local 20.20.20.20`
+Move the `cert.pem` and `key.pem` to `src/.certs` folder (create the folder if not present).
+
+### Required environmental variables
+
+copy `src/metax_api/settings/.env.template` as `src/metax_api/settings/.env` and fill required variables, you can find examples in ENV_VARS.md
+
+### Create log directory 
+
+`mkdir -p src/log/errors`
+
+### Run Metax inside a container (Optional)
+
+Check the IP addresses of Redis, RabbitMQ, ElasticSearch and Postgres:9 either from Portainer container list (click the link in the container name to see all attributes) or by going to portainer network tab or by typing `docker container ps` followed by `docker network inspect bridge`
+
+Build new docker image from repository root with this command (change ip-addresses to real ones:
+
+`docker build -t metax-api:latest --build-arg METAX_DATABASE_HOST=xxx.xx.x.x --build-arg REDIS_HOST=xxx.xx.x.x --build-arg RABBITMQ_HOST=xxx.xx.x.x --build-arg ELASTIC_SEARCH_HOST=xxx.xx.x.x:xxxx .`
+
+Run the built container with command:
+
+`docker run -it --name metax-web --mount type=bind,source="$(pwd)"/src,target=/code -p 8008:8008 --rm metax-api:latest`
+
+You should see metax-server starting at port 8008 with hot reload enabled
+
+### Initial setup commands
+docker network inspect bridge
+IF you configured metax-container, access the command line of the container with `docker exec -it metax-web bash`
+
+__NOTICE: Skip activating virtualenv and navigating to src folder if you have metax running on container__
+
+Activate your python 3.6 virtualenv, `cd` into `src` folder and run following commands:
+
+setup the database with migrate command:
+
+`python manage.py migrate`
+
+__NOTICE: Skip following steps if your running metax on container and have terminal open in the container__
+
+start the development server with:
+`python manage.py runsslserver --certificate .certs/cert.pem --key .certs/key.pem 8008`
+
+Open another terminal and `cd` into `src`, and load the initial data with following commands: 
+
+__These commands must be run in both setups (container/not-container metax)__
+
+`python manage.py index_refdata`
+
+`python manage.py reload_refdata_cache`
+
+`python manage.py loadinitialdata`
+
+`python manage.py loaddata metax_api/tests/testdata/test_data.json` 
+
+run the tests with command `DJANGO_ENV=test python manage.py test --failfast --keepdb -v 0`
+
+Metax api is available from your browser at https://localhost:8008
+
+
+[1]: https://docs.docker.com/engine/install/
+[2]: https://github.com/FiloSottile/mkcert
+