From a05aece41b2b1a8544632172389e8717eedfc6e4 Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Thu, 10 Dec 2020 12:41:51 +0200 Subject: [PATCH] Document env vars (#720) * document env vars part 1 * update .env.template to include only necessary env vars. Co-authored-by: Toni Nurmi --- ENV_VARS.md | 89 ++++++++++++------- src/metax_api/settings/.env.template | 35 +------- src/metax_api/settings/__init__.py | 23 +++-- .../settings/components/externals.py | 10 --- src/metax_api/settings/components/redis.py | 25 ++++-- src/metax_api/settings/components/rems.py | 20 +++++ 6 files changed, 117 insertions(+), 85 deletions(-) create mode 100644 src/metax_api/settings/components/rems.py diff --git a/ENV_VARS.md b/ENV_VARS.md index 87890e11..2b4a4a2e 100755 --- a/ENV_VARS.md +++ b/ENV_VARS.md @@ -1,33 +1,62 @@ # Required environmental variables -copy .env.template to .env and fill following variables manually - -| Name | Description | example | -| ------------------------------ | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------- | -| DATACITE_PREFIX | | | -| DATACITE_URL | | | -| DATACITE_ETSIN_URL_TEMPLATE | | | -| DATACITE_USERNAME | | | -| DATACITE_PASSWORD | | | -| DJANGO_SECRET_KEY | replace with proper django secret key | | -| ERROR_FILES_PATH | | /home/user/repo-root/logs/errors | -| ES_CONFIG_DIR | | /home/user/repo-root/src/metax_api/tasks/refdata/refdata_indexer/resources/es-config/ | -| LOCAL_REF_DATA_FOLDER | | /home/user/repo-root/src/metax/metax-api/src/metax_api/tasks/refdata/refdata_indexer/resources/local-refdata/ | -| LOGGING_DEBUG_HANDLER_FILE | | /home/user/repo-root/src/metax/metax-api/logs/metax_api.log | -| LOGGING_GENERAL_HANDLER_FILE | | /home/user/repo-root/src/metax/metax-api/logs/metax_api.log | -| LOGGING_JSON_FILE_HANDLER_FILE | | /home/user/repo-root/src/metax/metax-api/logs/metax_api.json.log | -| METAX_DATABASE | postgres database, must be created first | metax | -| METAX_DATABASE_PASSWORD | postgres owner of the database | | -| METAX_DATABASE_PORT | | 5432 | -| METAX_DATABASE_USER | | username | -| OAI_BASE_URL | | | -| ORG_FILE_PATH | | /home/user/repo-root/src/metax_api/tasks/refdata/refdata_indexer/resources/organizations/organizations.csv | -| RABBIT_MQ_PASSWORD | | guest | -| RABBIT_MQ_USER | | guest | -| REDIS_LOCALHOST_PORT | | 6379 | -| REDIS_PASSWORD | | | -| REMS_ETSIN_URL_TEMPLATE | | | -| REMS_FORM_ID | | | -| VALIDATE_TOKEN_URL | | | -| WKT_FILENAME | | /home/user/repo-root/src/metax_api/tasks/refdata/refdata_indexer/resources/uri_to_wkt.json | +copy .env.template to .env and fill the required values from below table. Required column tells if you have to have the variable in the .env file +| Name | Required | Default | Description | +| ------------------------------ | -------- | ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| ADDITIONAL_USER_PROJECTS_PATH | unknown | "" | No clue if this is important | +| ALLOWED_HOSTS | no | | defines which IP-addresses are allowed to access metax, DJANGO_ENV=local overrides this | +| AUTH_SERVER_LOGOUT_URL | unknown | | Requires testing if this is needed | +| DATACITE_ETSIN_URL_TEMPLATE | yes | | +| DATACITE_PASSWORD | yes | | +| DATACITE_PREFIX | yes | | +| DATACITE_URL | yes | | +| DATACITE_USERNAME | yes | | +| DEBUG | no | False | +| DJANGO_ENV | no | local | Specifies the environment, corresponds with the environments found in src/metax_api/settings/environments/ | +| DJANGO_SECRET_KEY | yes | | +| DRAFT_ENABLED | no | | +| ELASTIC_SEARCH_HOSTS | yes | | Elastic Search instance IP and port | +| ELASTIC_SEARCH_PORT | no | 9200 | Is not used currently, but should be in the future | +| ELASTIC_SEARCH_USE_SSL | yes | | Should Elastic Search queries use https | +| ERROR_FILES_PATH | no | src/log/errors | Error file folder | +| ES_CONFIG_DIR | no | src/metax_api/tasks/refdata/refdata_indexer/resources/es-config | metax-ops compatibility | +| LOCAL_REF_DATA_FOLDER | no | src/metax_api/tasks/refdata/refdata_indexer/resources/local-refdata | metax-ops compatibility | +| LOGGING_DEBUG_HANDLER_FILE | no | src/log/metax_api.log | metax-ops compatibility | +| LOGGING_GENERAL_HANDLER_FILE | no | src/log/metax_api.log | metax-ops compatibility | +| LOGGING_JSON_FILE_HANDLER_FILE | no | src/log/metax_api.json.log | metax-ops compatibility | +| METAX_API_ROOT | yes | | Metax internal variable, must be https | +| METAX_DATABASE | yes | | Postgres database name | +| METAX_DATABASE_PASSWORD | yes | | Postgres database password | +| METAX_DATABASE_PORT | no | 5432 | Postgres instance exposed port | +| METAX_DATABASE_USER | yes | | Postgres user which owns the database | +| METAX_ENV | no | local_development | Used by Metax internally, should be replaced with DJANGO_ENV in the future | +| OAI_BASE_URL | yes | | +| ORG_FILE_PATH | yes | src/metax_api/tasks/refdata/refdata_indexer/resources/organizations/organizations.csv | metax-ops compatibility | +| RABBIT_MQ_HOSTS | yes | | RabbitMQ instance IP | +| RABBIT_MQ_PASSWORD | no | guest | +| RABBIT_MQ_PORT | no | 5672 | +| RABBIT_MQ_TTV_ENABLED | no | | +| RABBIT_MQ_USER | no | guest | +| RABBIT_MQ_VHOST | no | | +| RABBIT_MQ_VHOST_TTV | no | | +| REDIS_HOST | yes | | Redis instance IP | +| REDIS_LOCALHOST_PORT | unknown | 6379 | Not sure if all references to this are gone | +| REDIS_PASSWORD | no | | +| REDIS_PORT | no | 6379 | +| REDIS_TEST_DB | yes | | Pick a number, any number | +| REDIS_USE_PASSWORD | no | false | +| REMS_API_KEY | no | +| REMS_AUTO_APPROVER | no | +| REMS_BASE_URL | no | +| REMS_ENABLED | no | +| REMS_ETSIN_URL_TEMPLATE | yes | | +| REMS_FORM_ID | yes | | +| REMS_METAX_USER | no | +| REMS_REPORTER_USER | no | +| SERVER_DOMAIN_NAME | no | +| TRAVIS | no | false | Metax internal used to figure out if Metax is running on Travis virtual machine | +| V1_ENABLED | no | | +| V2_ENABLED | no | | +| VALIDATE_TOKEN_URL | yes | | +| WKT_FILENAME | no | src/metax_api/tasks/refdata/refdata_indexer/resources/uri_to_wkt.json | \ No newline at end of file diff --git a/src/metax_api/settings/.env.template b/src/metax_api/settings/.env.template index 7c990a84..dc8b7500 100755 --- a/src/metax_api/settings/.env.template +++ b/src/metax_api/settings/.env.template @@ -1,42 +1,15 @@ -ALLOWED_HOSTS=metax.csc.local,20.20.20.20 AUTH_SERVER_LOGOUT_URL= DATACITE_PREFIX= DATACITE_URL= +DATACITE_PASSWORD= DATACITE_ETSIN_URL_TEMPLATE= DATACITE_USERNAME= -DATACITE_PASSWORD= DEBUG=on -DJANGO_ENV=local DJANGO_SECRET_KEY= DRAFT_ENABLED=true -ELASTIC_SEARCH_HOSTS=http://localhost:9200/ -ELASTIC_SEARCH_PORT=9200 -ELASTIC_SEARCH_USE_SSL=False -FLUSH_PASSWORD= -METAX_API_ROOT= METAX_DATABASE= -METAX_DATABASE_HOST=localhost METAX_DATABASE_PASSWORD= METAX_DATABASE_USER= -OAI_BASE_URL=http: -RABBIT_MQ_HOSTS= -RABBIT_MQ_PASSWORD= -RABBIT_MQ_PORT= -RABBIT_MQ_USER= -RABBIT_MQ_VHOST=metax -REDIS_HOST= -REDIS_PASSWORD= -REDIS_PORT= -REDIS_SENTINEL_SERVICE= -REDIS_TEST_DB=15 -REDIS_USE_PASSWORD=false -REMS_API_KEY= -REMS_AUTO_APPROVER= -REMS_BASE_URL= -REMS_ENABLED= -REMS_ETSIN_URL_TEMPLATE= -REMS_FORM_ID= -REMS_METAX_USER= -REMS_REPORTER_USER= -SERVER_DOMAIN_NAME= -VALIDATE_TOKEN_URL= \ No newline at end of file +OAI_BASE_URL= +RABBIT_MQ_VHOST= +REMS_ENABLED=false \ No newline at end of file diff --git a/src/metax_api/settings/__init__.py b/src/metax_api/settings/__init__.py index fc981768..97e93820 100755 --- a/src/metax_api/settings/__init__.py +++ b/src/metax_api/settings/__init__.py @@ -20,28 +20,32 @@ env = environ.Env( # set casting, default value ADDITIONAL_USER_PROJECTS_PATH=(str, ""), + ALLOWED_HOSTS=(list, ["metax.csc.local", "20.20.20.20"]), DEBUG=(bool, False), DJANGO_ENV=(str, "local"), ELASTIC_SEARCH_PORT=(int, 9200), + ELASTIC_SEARCH_USE_SSL=(bool, False), ERROR_FILES_PATH=(str, join(BASE_DIR, "log", "errors")), ES_CONFIG_DIR=(str, join(REFDATA_INDEXER_PATH, "resources", "es-config/")), - LOCAL_REF_DATA_FOLDER=( - str, - join(REFDATA_INDEXER_PATH, "resources", "local-refdata/"), - ), + LOCAL_REF_DATA_FOLDER=(str,join(REFDATA_INDEXER_PATH, "resources", "local-refdata/"),), LOGGING_DEBUG_HANDLER_FILE=(str, join(BASE_DIR, "log", "metax_api.log")), LOGGING_GENERAL_HANDLER_FILE=(str, join(BASE_DIR, "log", "metax_api.log")), LOGGING_JSON_FILE_HANDLER_FILE=(str, join(BASE_DIR, "log", "metax_api.json.log")), - METAX_ENV=(str, "local_development"), + METAX_DATABASE_HOST=(str, "localhost"), METAX_DATABASE_PORT=(str, 5432), - ORG_FILE_PATH=( - str, - join(REFDATA_INDEXER_PATH, "resources", "organizations", "organizations.csv"), - ), + METAX_ENV=(str, "local_development"), + METAX_API_ROOT=(str, "https://localhost:8008"), + ORG_FILE_PATH=(str, join(REFDATA_INDEXER_PATH, "resources", "organizations", "organizations.csv"),), RABBIT_MQ_PORT=(int, 5672), REDIS_HOST=(str, "localhost"), + RABBIT_MQ_PASSWORD=(str, "guest"), + RABBIT_MQ_USER=(str, "guest"), REDIS_PORT=(int, 6379), + REDIS_TEST_DB=(int, 15), + REDIS_USE_PASSWORD=(bool, False), + SERVER_DOMAIN_NAME=(str, "metax.csc.local"), TRAVIS=(bool, False), + VALIDATE_TOKEN_URL=(str, "https://127.0.0.1/secure/validate_token"), WKT_FILENAME=(str, join(REFDATA_INDEXER_PATH, "resources", "uri_to_wkt.json")), ) # reading .env file @@ -57,6 +61,7 @@ "components/elasticsearch.py", "components/rabbitmq.py", "components/externals.py", + "components/rems.py", "environments/{0}.py".format(ENV), # Optionally override some settings: # optional('environments/legacy.py'), diff --git a/src/metax_api/settings/components/externals.py b/src/metax_api/settings/components/externals.py index 0720d903..b8077e00 100755 --- a/src/metax_api/settings/components/externals.py +++ b/src/metax_api/settings/components/externals.py @@ -23,16 +23,6 @@ "PREFIX": env("DATACITE_PREFIX"), "URL": env("DATACITE_URL"), } -REMS = { - "ENABLED": env("REMS_ENABLED"), - "API_KEY": env("REMS_API_KEY"), - "BASE_URL": env("REMS_BASE_URL"), - "ETSIN_URL_TEMPLATE": env("REMS_ETSIN_URL_TEMPLATE"), - "METAX_USER": env("REMS_METAX_USER"), - "REPORTER_USER": env("REMS_REPORTER_USER"), - "AUTO_APPROVER": env("REMS_AUTO_APPROVER"), - "FORM_ID": int(env("REMS_FORM_ID")), -} ORG_FILE_PATH = env("ORG_FILE_PATH") WKT_FILENAME = env("WKT_FILENAME") LOCAL_REF_DATA_FOLDER = env("LOCAL_REF_DATA_FOLDER") diff --git a/src/metax_api/settings/components/redis.py b/src/metax_api/settings/components/redis.py index f663507a..b26020ef 100755 --- a/src/metax_api/settings/components/redis.py +++ b/src/metax_api/settings/components/redis.py @@ -1,7 +1,10 @@ +from environ import ImproperlyConfigured +import logging from metax_api.settings import env +logger = logging.getLogger(__name__) + REDIS = { - "PASSWORD": env("REDIS_PASSWORD"), "LOCALHOST_PORT": env("REDIS_PORT"), "HOST": env("REDIS_HOST"), "PORT": env("REDIS_PORT"), @@ -11,9 +14,21 @@ "TEST_DB": env("REDIS_TEST_DB"), # enables extra logging to console during cache usage "DEBUG": False, - "SENTINEL": { - "HOSTS": [["127.0.0.1", 16379], ["127.0.0.1", 16380], ["127.0.0.1", 16381]], - "SERVICE": env("REDIS_SENTINEL_SERVICE"), - }, } REDIS_USE_PASSWORD = env("REDIS_USE_PASSWORD") +REDIS_USE_SENTINEL = False + +if REDIS_USE_PASSWORD: + try: + REDIS["PASSWORD"] = env("REDIS_PASSWORD") + except ImproperlyConfigured as e: + logger.warning(e) + +if REDIS_USE_SENTINEL: + try: + REDIS["SENTINEL"] = { + "HOSTS": [["127.0.0.1", 16379], ["127.0.0.1", 16380], ["127.0.0.1", 16381]], + "SERVICE": env("REDIS_SENTINEL_SERVICE"), + } + except ImproperlyConfigured as e: + logger.warning(e) diff --git a/src/metax_api/settings/components/rems.py b/src/metax_api/settings/components/rems.py new file mode 100644 index 00000000..2b5ee3fa --- /dev/null +++ b/src/metax_api/settings/components/rems.py @@ -0,0 +1,20 @@ +from environ import ImproperlyConfigured +import logging + +logger = logging.getLogger(__name__) + +from metax_api.settings import env +REMS = { + "ENABLED": env("REMS_ENABLED"), +} +if REMS["ENABLED"]: + try: + REMS["API_KEY"]: env("REMS_API_KEY") + REMS["BASE_URL"]: env("REMS_BASE_URL") + REMS["ETSIN_URL_TEMPLATE"]: env("REMS_ETSIN_URL_TEMPLATE") + REMS["METAX_USER"]: env("REMS_METAX_USER") + REMS["REPORTER_USER"]: env("REMS_REPORTER_USER") + REMS["AUTO_APPROVER"]: env("REMS_AUTO_APPROVER") + REMS["FORM_ID"]: int(env("REMS_FORM_ID")) + except ImproperlyConfigured as e: + logger.warning(e) \ No newline at end of file