Merge pull request #1599 from ucfopen/dev/10.3.0

Dev/10.3.0
ucfopen · Dec 12, 2024 · a6a895a · a6a895a
2 parents 85dd532 + a0ebbf7
commit a6a895a
Show file tree

Hide file tree

Showing 187 changed files with 7,754 additions and 1,704 deletions.
diff --git a/.env b/.env
@@ -1,19 +1,25 @@
-# =========================
+# =================================
 # CONFIGURE MATERIA
-# 1. Copy `.env` to `.env.local` (make changes in .env.local)
-# 2. Set every `<MUST_SET>` value
-# 3. Comb through for any other settings you may wish to change
-# 4. Visit Materia in browser, check logs in fuel/app/logs for debugging
 #
-# DETAILED INSTRUCTIONS
-# !!! View the Materia README for options, descriptions & permitted values !!!
+# This .env file is not referenced by the default development & nondev docker stack! Use docker/.env.local instead.
+# It's recommended to use this .env file as a template for production.
+# Your docker compose file(s) will need to be updated to reference your final .env and its relative path in the env_file: directive of each service.
+#
+# DETAILED INSTRUCTIONS FOR PRODUCTION USE
 #
-# Only `BOOL_` options become boolean values, and ONLY `true` evaluates to true
+# 1. If coming from a nondev instance, copy relevant values from docker/.env.local
+# 2. Set every `<MUST_SET>` value
+# 3. Review remaining optional settings
+# 4. Update your docker compose file(s) to point to the final .env file and location. Services should use the `env_file:` directive to include env vars as required
+# 5. Remove any dev `environment:` references in compose file(s) if they are present
+# 6. Start docker containers (or stop, rm, and restart if already running)
+# 7. Visit Materia in browser, check logs in fuel/app/logs for debugging
 #
-# Docker/Dev notes:
-# docker/config/materia-docker.env.local is loaded instead of .env.local
+# Refer to docker/README.md for more information about environment configs
+#
+# !! NOTE: Only `BOOL_` options become boolean values, and ONLY `true` evaluates to true
 
-# GENERAL ===================
+# GENERAL ==========================
 
 FUEL_ENV=production
 #DATABASE_URL=<MUST_SET>
@@ -24,38 +30,46 @@ BOOL_SEND_EMAILS=false
 #FUEL_ALWAYS_LOAD_MODULES=""
 #GOOGLE_ANALYTICS_ID=xxx
 
-# LOGGING ===================
+# LOGGING ==========================
 
 #FUEL_LOG_THRESHOLD=300
 #LOG_HANDLER=DEFAULT
 
-# ASSETS ===================
+# ASSETS ===========================
 
-#URLS_STATIC=
-#URLS_ENGINES=
+#URLS_STATIC= 								# Set if static files and assets come from a different domain. While used in dev, not required for prod
+#URLS_ENGINES= 								# same as above
 #BOOL_ADMIN_UPLOADER_ENABLE=true
-#ASSET_STORAGE_DRIVER=file
-#ASSET_STORAGE_S3_REGION=us-east-1
-#ASSET_STORAGE_S3_BUCKET=
-#ASSET_STORAGE_S3_BASEPATH=
-#ASSET_STORAGE_S3_KEY=
-#ASSET_STORAGE_S3_SECRET=
+ASSET_STORAGE_DRIVER=file					# file | s3 | db (db not recommended)
+
+# AWS S3 ===========================
+# Note: the <MUST_SET> options here are only required if S3 storage driver is enabled
 
-# SESSION & CACHE ===================
+# ASSET_STORAGE_S3_CREDENTIAL_PROVIDER=		# env | imds
+# ASSET_STORAGE_S3_BUCKET=<MUST_SET>
+# ASSET_STORAGE_S3_ENDPOINT=				# not required for S3 on AWS
+# ASSET_STORAGE_S3_KEY=<MUST_SET>
+# ASSET_STORAGE_S3_SECRET=<MUST_SET>
+# ASSET_STORAGE_S3_REGION=
+# ASSET_STORAGE_S3_BUCKET=
+# ASSET_STORAGE_S3_BASEPATH=
+# AWS_SESSION_TOKEN=<MUST_SET> 				# STS token for s3 development. Not required if using imds
+
+# SESSION & CACHE ==================
 
 #MEMCACHED_HOST=localhost
 #MEMCACHED_PORT=11211
-#CACHE_DRIVER=file
-#SESSION_DRIVER=file
+#CACHE_DRIVER=file							# memcached | file
+#SESSION_DRIVER=file						# memcached | file | db
 SESSION_EXPIRATION=21600
 
-# THEME ===================
+# THEME ============================
 
 #THEME_ACTIVE=default
 # relative to /fuel/packages/
 #THEME_PACKAGE=materia-theme-ucf
 
-# AUTH ===================
+# AUTH ============================
 
 #AUTH_DRIVERS=Materiaauth
 #AUTH_SALT=<MUST_SET>
@@ -67,14 +81,14 @@ SESSION_EXPIRATION=21600
 #USER_INSTRUCTOR_PASSWORD
 #USER_STUDENT_PASSWORD
 
-# CRYPTO ===================
+# CRYPTO ==========================
 
 #CRYPTO_KEY=<MUST_SET>
 #CRYPTO_IV=<MUST_SET>
 #CRYPTO_HMAC=<MUST_SET>
 #CIPHER_KEY=<MUST_SET>
 
-# LTI ===================
+# LTI ============================
 
 #BOOL_LTI_RESTRICT_LOGINS_TO_LAUNCHES=false
 #LTI_GUID=
@@ -88,3 +102,19 @@ LTI_KEY="materia-production-lti-key"
 #BOOL_LTI_USE_LAUNCH_ROLES=true
 #BOOL_LTI_GRACEFUL_CONFIG_FALLBACK=true
 #BOOL_LTI_LOG_FOR_DEBUGGING=false
+
+# QUESTION GENERATION ===========
+# Note: <MUST_SET> options are only required if GENERATION_ENABLED is true
+
+#GENERATION_ENABLED=true
+#GENERATION_ALLOW_IMAGES=false
+#GENERATION_API_PROVIDER=<MUST_SET>			# openai | azure_openai
+#GENERATION_API_KEY=<MUST_SET>
+#GENERATION_API_ENDPOINT=					# required for azure
+#GENERATION_API_VERSION=					# required for azure
+#GENERATION_API_MODEL=						# required for openai
+#GENERATION_LOG_STATS=true					# optional. stats are logged to debug threshold.
+
+# WEBSERVER =====================
+
+#IS_SERVER_HTTPS=true						# defaults to true, only set to false if required
diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml
@@ -8,7 +8,7 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Run Tests
-      run: cd docker && ./run_tests_ci.sh
+      run: cd docker && touch .env.local && ./run_tests_ci.sh
 
   docker_images_and_release_zip:
     needs: ci_tests
@@ -29,17 +29,40 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Build App Image
+      - name: Create .env.local to satisfy build requirements
         run: |
           cd docker
-          docker-compose build --no-cache webserver app
+          if [ ! -f .env.local ]; then
+            touch .env.local
+          fi
 
-      - name: Push App Images
+      - name: Build App and Webserver Images
         run: |
-          docker tag ucfopen/materia:app-dev ghcr.io/${{ github.repository_owner }}/materia:app-${{ github.sha }}
-          docker tag ucfopen/materia:app-dev ghcr.io/${{ github.repository_owner }}/materia:app-${{ steps.tag_name.outputs.GIT_TAG }}
-          docker tag ucfopen/materia:webserver-dev ghcr.io/${{ github.repository_owner }}/materia:webserver-${{ github.sha }}
-          docker tag ucfopen/materia:webserver-dev ghcr.io/${{ github.repository_owner }}/materia:webserver-${{ steps.tag_name.outputs.GIT_TAG }}
+          cd docker
+          docker compose build --no-cache webserver app
+          docker compose -f docker-compose.yml -f docker-compose.development.yml build --no-cache fakes3
+
+      - name: Push Dev App and Webserver Images
+        if: ${{ startsWith(github.ref, 'refs/tags/v') && (contains(github.ref, '-alpha') || contains(github.ref, '-rc')) }}
+        run: |
+          docker push ghcr.io/${{ github.repository_owner }}/materia:app-dev
+          docker push ghcr.io/${{ github.repository_owner }}/materia:webserver-dev
+          docker push ghcr.io/${{ github.repository_owner }}/materia:fake-s3-dev
+
+      - name: Push Stable App and Webserver Images
+        if: ${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-alpha') && !contains(github.ref, '-rc') }}
+        run: |
+          docker tag ghcr.io/${{ github.repository_owner }}/materia:app-dev ghcr.io/${{ github.repository_owner }}/materia:app-stable
+          docker tag ghcr.io/${{ github.repository_owner }}/materia:webserver-dev ghcr.io/${{ github.repository_owner }}/materia:webserver-stable
+          docker push ghcr.io/${{ github.repository_owner }}/materia:app-stable
+          docker push ghcr.io/${{ github.repository_owner }}/materia:webserver-stable
+
+      - name: Push Versioned App and Webserver Images
+        run: |
+          docker tag ghcr.io/${{ github.repository_owner }}/materia:app-dev ghcr.io/${{ github.repository_owner }}/materia:app-${{ github.sha }}
+          docker tag ghcr.io/${{ github.repository_owner }}/materia:app-dev ghcr.io/${{ github.repository_owner }}/materia:app-${{ steps.tag_name.outputs.GIT_TAG }}
+          docker tag ghcr.io/${{ github.repository_owner }}/materia:webserver-dev ghcr.io/${{ github.repository_owner }}/materia:webserver-${{ github.sha }}
+          docker tag ghcr.io/${{ github.repository_owner }}/materia:webserver-dev ghcr.io/${{ github.repository_owner }}/materia:webserver-${{ steps.tag_name.outputs.GIT_TAG }}
           docker push ghcr.io/${{ github.repository_owner }}/materia:app-${{ github.sha }}
           docker push ghcr.io/${{ github.repository_owner }}/materia:app-${{ steps.tag_name.outputs.GIT_TAG }}
           docker push ghcr.io/${{ github.repository_owner }}/materia:webserver-${{ github.sha }}
@@ -63,7 +86,7 @@ jobs:
           overwrite: true
 
       - name: Upload to Pre-Release
-        if: ${{ startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-alpha') && contains(github.ref, '-rc') }}
+        if: ${{ startsWith(github.ref, 'refs/tags/v') && contains(github.ref, '-alpha') || contains(github.ref, '-rc') }}
         uses: svenstaro/upload-release-action@v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.gitignore b/.gitignore
@@ -14,6 +14,7 @@ nbproject/
 
 # custom config
 .env.local
+.env.local.mysql
 
 # hidden files created by Windows
 Thumbs.db
@@ -85,6 +86,7 @@ public/js/materia.storage.table.js
 public/js/student.js
 public/js/vendor/*
 
+public/openai_usage.txt
 
 # Installed Widgets
 public/widget
@@ -122,3 +124,9 @@ public/dist/*
 !public/dist/package.json
 !public/dist/path.js
 !public/dist/README.md
+
+public/js
+public/css
+
+# docker override is now untracked as of v10.3.0
+docker/docker-compose.override.yml
diff --git a/README.md b/README.md
@@ -22,41 +22,54 @@ While casual references to _Materia_ typically involve both the platform and its
 
 ## Installation
 
-Materia is configured to use Docker containers in production environments, orchestrated through docker compose, though other orchestration frameworks could potentially be used instead. While it may be possible to deploy Materia without Docker, we **do not recommend doing so**.
+Materia is configured to use docker containers in production environments, orchestrated through docker compose, though other orchestration frameworks could potentially be used instead. While it may be possible to deploy Materia without docker, we **do not recommend doing so**.
 
 ### Docker Deployment
 
-We publish production ready docker and nginx containers in the [Materia Docker repository](https://github.com/orgs/ucfopen/packages/container/package/materia).  For more info on using Docker in Production, read the [Materia Docker Readme](docker/README.md)
+Refer to the [Materia docker readme](docker/README.md) for a full breakdown of Materia's docker configuration and deployment in both development and production.
 
 ### Configuration
 
-Visit the [Server Variables](https://ucfopen.github.io/Materia-Docs/admin/server-variables.html) page on our docs site for information about configuration through environment variables.
+Materia uses environment variables to facilitate application and webserver configuration. These are typically sourced from `.env` or `.env.local` files on the host machine that are volume mounted into their associated containers.
 
-## Development
+Visit docker README as well as the [Server Variables](https://ucfopen.github.io/Materia-Docs/admin/server-variables.html) page on our docs site for information about configuration through environment variables. The [root env file](.env) serves as a configuration template for production instances of Materia.
 
-Code contributors should review the [CONTRIBUTING](CONTRIBUTING.md) document before proceeding.
+## Setup
+
+Materia provides a pair of setup scripts for out-of-the-box deployment based on two different use cases:
+
+1. `run_first_for_dev.sh` sets up and configures your local instance of Materia for development. This includes additional volume mounts for project files and makes use of additional containers for mysql, s3, and memcached.
+2. `run_first_for_nondev.sh` is ideal for users who just want to explore Materia locally and potentially transition to a production instance. The script dynamically configures the override compose file based on selections you make in the script.
 
-### Local Dev with Docker
+> [!NOTE]
+> `yq` is required for the nondev script. This may come preinstalled in some OS distributions but you should ensure the correct version is installed. Consult the [yq installation guide](https://github.com/mikefarah/yq?tab=readme-ov-file#install) for more info.
 
-Get started with a local dev server:
+In either case, first-time setup involves the following:
 
 ```
 git clone https://github.com/ucfopen/Materia.git
-
 cd Materia/docker
+```
 
-./run_first.sh
+Followed by either:
+```
+./run_first_for_dev.sh
 ```
+For local development or 
 
-The `run_first.sh` script only has to be run once for initial setup. Afterwards, your local copy will persist in a docker volume unless you explicitly use `docker-compose down` or delete the volume manually.
+```
+./run_first_for_nondev.sh
+```
+For creating a local instance where development is not desired.
 
-Use `docker-compose up` to run your local instance. The compose process must persist to keep the application alive. Materia is configured to run at `https://127.0.0.1` by default.
+The `run_first` scripts only have to be run once for initial setup. Afterwards, your local copy will persist in a docker volume unless you explicitly use `docker compose down` or delete the volumes manually.
 
-In a separate terminal window, run `yarn dev` to enable the webpack dev server and live reloading while making changes to JS and CSS assets. 
+Use `docker compose up` to run your local instance. The compose process must persist to keep the application alive. Materia is configured to run at `https://localhost` by default.
 
-Note that Materia uses a self-signed certificate to facilitate https traffic locally. Your browser may require security exceptions for both `127.0.0.1:443` and `127.0.0.1:8008`.
+Note that Materia uses a self-signed certificate to facilitate https traffic locally. Your browser may require security exceptions for your application on ports `443` and `8008` (if setup for local development).
 
-More info about Materia Docker can be found in the [Materia Docker Readme](docker/README.md)
+> [!NOTE]
+> If local development of static assets (JS and CSS) is desired, run `yarn dev` in a separate terminal window to enable the webpack dev server and live reloading. This requires node and yarn to be installed on the host machine.
 
 ### Creating additional users
 
@@ -66,20 +79,14 @@ See the wiki page for [Creating a local user](https://github.com/ucfopen/Materia
 
 Tests run in the docker environment to maintain consistency. View the `run_tests_*.sh` scripts in the docker directory for options.
 
-#### Running A Single Test Group
-
-Inspect the actual test command in `/.run_tests.sh` for guidance, but as of the time of writing this, you can run a subset of the tests in the docker environment to save time.
-
-The following command will run just the **Oauth** tests rather quickly:
-
-```
-./run_tests.sh --group=Oauth
-```
-
 ### Git Hooks
 
 There is a pre-commit hook available to ensure your code follows our linting standards. Check out the comments contained inside the hook files (in the githooks directory) to install it, you may need a few dependencies installed to get linting working.
 
+## Contributing
+
+Code contributors should review the [CONTRIBUTING](CONTRIBUTING.md) document before proceeding.
+
 ## Authentication
 
 Materia supports two forms of authentication:
@@ -89,4 +96,57 @@ Materia supports two forms of authentication:
 
 ## Asset Storage
 
-Materia enables users to upload media assets for their widgets, including images and audio. There are two asset storage drivers available out of the box: `file` and `db`. `file` is the default asset storage driver, which can be explicitly set via the `ASSET_STORAGE_DRIVER` environment variable.
+Users can upload media assets (images and audio) for use in their widgets, facilitated through a media importer that is provided by Materia itself. Asset storage drivers include:
+
+- `file`: Assets are stored on the local filesystem of the application. It is recommended that assets are backed up and synced with an external storage solution (such as S3) to ensure the files persist across application instances.
+- `s3`: Files are uploaded to and requested directly from AWS S3. This is the most straightforward and recommended storage driver option. Be sure to consult the [Materia Docker Readme](docker/README.md) for additional environment variables associated with using S3.
+- `db`: This storage driver stores asset binaries directly in the database. This option allows Materia to run on cloud hosting options with very limited storage volumes. The `db` storage driver option is not recommended for general use.
+
+> [!WARNING]
+> The `db` asset storage driver option is deprecated and will be removed in the next major version of Materia.
+
+The storage driver is configured via the `ASSET_STORAGE_DRIVER` environment variable.
+
+### Local Asset Storage With S3
+
+A `fakes3` container is instantiated as part of the default development stack and the `ASSET_STORAGE_DRIVER` environment variable is set to `s3` by default in the development `.env` file located in `docker/.env`. When using `fakes3`, this is all that is required to simulate S3 usage locally.
+
+To use an actual S3 bucket for local dev:
+
+1. Set `DEV_ONLY_FAKES3_DISABLED` environment variable in `docker/.env` to `true`
+2. Set `ASSET_STORAGE_S3_BUCKET` to your bucket name
+3. Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` in `.env.local`. (Tip: You can run `aws configure export-credentials --profile YOUR_PROFILE_NAME --format env-no-export` to get these)
+
+> [!NOTE]
+> Note that `fakes3` asset storage is disabled when `FUEL_ENV` is set to `production`.
+
+## Widget Management
+
+A default list of widgets will be installed as part of the first-time setup process, but the widget ecosystem has many more! Peruse the [Materia Widget Gallery](https://ucfopen.github.io/materia-widget-gallery/) to view additional widgets to install.
+
+### Installing Widgets
+
+Widgets can be installed in one of two ways:
+
+1. A user with the `super_user` role can visit the Widget Admin panel by navigating to `your.materia.url/admin/widget` or by selecting the orange "Admin" button at the top. Select a `.wigt` file from the file upload dialog to install it.
+2. Widgets can be installed from the cli on the application container once running:
+
+```
+$ docker exec -it <container name or id> sh
+$ wget url/for/materia/widget.wigt
+$ php oil r widget:install widget.wigt
+```
+
+The `.wigt` files do not need to be retained once a widget is installed.
+
+## Theming
+
+Theme overrides are facilitated through FuelPHP packages, installed via composer. Materia ships with `Materia-Theme-UCF` by default.
+
+Review the [repository README](https://github.com/ucfopen/Materia-Theme-UCF) for `Materia-Theme-UCF` for a breakdown of using theme overrides in Materia 10.x and later.
+
+## Transitioning to Production
+
+More information about creating a production-capable Materia instance can be found in the [Materia Docker Readme](docker/README.md).
+
+