Merge pull request #681 from lgpage/hidden-tests

Hide tests in "Autograder tests" cells

nbgrader/apps/assignapp.py

@@ -15,6 +15,7 @@
     SaveCells,
     CheckCellMetadata,
     ClearOutput,
+    ClearHiddenTests,
 )
 
 aliases = {}
@@ -36,6 +37,7 @@
     'no-metadata': (
         {
             'ClearSolutions': {'enforce_metadata': False},
+            'ClearHiddenTests': {'enforce_metadata': False},
             'CheckCellMetadata': {'enabled': False},
             'ComputeChecksums': {'enabled': False}
         },
@@ -47,6 +49,7 @@
     ),
 })
 
+
 class AssignApp(BaseNbConvertApp):
 
     name = u'nbgrader-assign'
@@ -144,8 +147,12 @@ def _output_directory(self):
         CheckCellMetadata,
         ComputeChecksums,
         SaveCells,
-        CheckCellMetadata
+        ClearHiddenTests,
+        ComputeChecksums,
+        CheckCellMetadata,
     ])
+    # NB: ClearHiddenTests must come after ComputeChecksums and SaveCells.
+    # ComputerChecksums must come again after ClearHiddenTests.
 
     def build_extra_config(self):
         extra_config = super(AssignApp, self).build_extra_config()
@@ -224,4 +231,3 @@ def init_assignment(self, assignment_id, student_id):
             # part of the assignment, and if so, remove them
             if self.notebook_id == "*":
                 self._clean_old_notebooks(assignment_id, student_id)
-

nbgrader/docs/source/configuration/student_version.rst

@@ -4,26 +4,31 @@ Customizing how the student version of an assignment looks
 .. seealso::
 
     :doc:`/user_guide/creating_and_grading_assignments`
-        Documentation for ``nbgrader assign``, ``nbgrader autograde``, ``nbgrader formgrade``, and ``nbgrader feedback``.
+        Documentation for ``nbgrader assign``, ``nbgrader autograde``,
+        ``nbgrader formgrade``, and ``nbgrader feedback``.
 
     :doc:`/command_line_tools/nbgrader-assign`
         Command line options for ``nbgrader assign``
 
     :doc:`config_options`
         Details on ``nbgrader_config.py``
 
+"Autograded answer" cells
+-------------------------
+
 Default behavior
-----------------
+^^^^^^^^^^^^^^^^
 
-By default, ``nbgrader assign`` will replace regions beginning with ``### BEGIN
-SOLUTION`` and ``### END SOLUTION`` with:
+By default, ``nbgrader assign`` will replace regions beginning with
+``### BEGIN SOLUTION`` and ``### END SOLUTION`` with:
 
 .. code:: python
 
     # YOUR CODE HERE
     raise NotImplementedError
 
-Note that the code stubs will be properly indented based on the indentation of the solution delimeters. For example, if your original code is:
+Note that the code stubs will be properly indented based on the indentation of
+the solution delimeters. For example, if your original code is:
 
 .. code:: python
 
@@ -50,7 +55,7 @@ which is ``YOUR ANSWER HERE``.
 
 
 Changing the defaults
----------------------
+^^^^^^^^^^^^^^^^^^^^^
 
 If you need to change these defaults (e.g., if your class doesn't use Python,
 or isn't taught in English), the values can be configured in the
@@ -102,3 +107,83 @@ can be configured through the ``ClearSolutions.text_stub`` option:
 
     c.ClearSolutions.text_stub = "Please replace this text with your response."
 
+
+"Autograder tests" cells with hidden tests
+------------------------------------------
+
+.. versionadded:: 0.5.0
+
+Default behavior
+^^^^^^^^^^^^^^^^
+
+By default, ``nbgrader assign`` will remove tests wrapped within the
+``BEGIN HIDDEN TESTS`` and ``END HIDDEN TESTS`` comment delimeters, for
+example:
+
+.. code:: python
+
+    assert squares(1) = [1]
+    ### BEGIN HIDDEN TESTS
+    assert squares(2) = [1, 4]
+    ### END HIDDEN TESTS
+
+will be released as:
+
+.. code:: python
+
+    assert squares(1) = [1]
+
+These comment delimeters are independent of the programming language used and
+the number of comment characters used in the source notebook. For example, this
+default will work for both ``Python``:
+
+.. code:: python
+
+    assert squares(1) = [1]
+    ### BEGIN HIDDEN TESTS
+    assert squares(2) = [1, 4]
+    ### END HIDDEN TESTS
+
+and ``JavaScript``:
+
+.. code-block:: javascript
+
+    function assert(answer, expected, msg) {
+        correct = ...;  // validate the answer
+        if (!correct) {
+            throw msg || "Incorrect answer";
+        }
+    }
+
+    assert(squares(1), [1]);
+    // BEGIN HIDDEN TESTS
+    assert(squares(2), [1, 4]);
+    // END HIDDEN TESTS
+
+.. note::
+
+    Keep in mind that wrapping all tests (for an "Autograder tests" cell) in
+    this special syntax will remove all these tests in the release version and
+    the students will only see a blank cell. It is recommended to have at least
+    one or more visible tests, or a comment in the cell for the students to
+    see.
+
+Changing the defaults
+^^^^^^^^^^^^^^^^^^^^^
+
+If you need to change these defaults (e.g., if your class isn't taught in
+English), the values can be configured in the :doc:`nbgrader_config.py
+<config_options>` file. Most relevant are the options to the
+``ClearHiddenTests`` preprocessor, which is the part of nbgrader that actually
+removes the tests when producing the student version of the notebook.
+
+You can specify hidden test delimeters you want by setting the
+``ClearHiddenTests.begin_test_delimeter`` and
+``ClearHiddenTests.end_test_delimeter`` config options:
+
+.. code:: python
+
+    c = get_config()
+    c.ClearHiddenTests.begin_test_delimeter = "VERBORGE TOESTE BEGIN"
+    c.ClearHiddenTests.end_test_delimeter = "VERBORGE TOESTE EINDIG"
+

nbgrader/docs/source/user_guide/creating_and_grading_assignments.ipynb

@@ -219,6 +219,52 @@
     "is active; it will not be visible to students.*"
    ]
   },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    ".. _autograder-tests-cell-hidden-tests:"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### \"Autograder tests\" cells with hidden tests"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    ".. versionadded:: 0.5.0"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Tests in \"Autograder tests\" cells can be hidden through the use of a special syntax such as ``### BEGIN HIDDEN TESTS`` and ``### END HIDDEN TESTS``, for example:\n",
+    "\n",
+    "![](images/autograder_tests_hidden_tests.png)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "When creating the release version (see :ref:`assign-and-release-an-assignment`), the region between the special syntax lines will be removed. If this special syntax is not used, then the contents of the cell will remain as is. Please see :doc:`/configuration/student_version` for details on how to customize this behavior.\n",
+    "\n",
+    ".. note::\n",
+    "\n",
+    "    Keep in mind that wrapping all tests (for an \"Autograder tests\" cell) in this special syntax will remove all these tests in\n",
+    "    the release version and the students will only see a blank cell. It is recommended to have at least one or more visible \n",
+    "    tests, or a comment in the cell for the students to see.\n",
+    "\n",
+    "    These hidden tests are placed back into the \"Autograder tests\" cells when running ``nbgrader autograde``\n",
+    "    (see :ref:`autograde-assignments`)."
+   ]
+  },
   {
    "cell_type": "raw",
    "metadata": {},

nbgrader/docs/source/user_guide/faq.rst

@@ -16,9 +16,8 @@ notebooks, you can pass the ``--no-execute`` flag to
 Can I hide the test cells in a nbgrader assignment?
 ---------------------------------------------------
 
-Not at the moment, though it is on the todo list (see `#390
-<https://github.com/jupyter/nbgrader/issues/390>`_). :ref:`PRs welcome!
-<pull-request>`
+Yes, as of version 0.5.0 of ``nbgrader`` you will be able to hide tests
+in "Autograder tests" cells (see :ref:`autograder-tests-cell-hidden-tests`).
 
 How does nbgrader ensure that students do not change the tests?
 ---------------------------------------------------------------

nbgrader/docs/source/user_guide/managing_assignment_files.ipynb

@@ -783,6 +783,17 @@
     "![](images/assignment_list_validate_succeeded.png)"
    ]
   },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    ".. note::\n",
+    "\n",
+    "   If the notebook has been released with hidden tests removed from the source version\n",
+    "   (see :ref:`autograder-tests-cell-hidden-tests`) then this validation is only done against the tests the students can\n",
+    "   see in the release version."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

nbgrader/preprocessors/__init__.py

@@ -14,6 +14,7 @@
 from .limitoutput import LimitOutput
 from .deduplicateids import DeduplicateIds
 from .latesubmissions import AssignLatePenalties
+from .clearhiddentests import ClearHiddenTests
 
 __all__ = [
     "AssignLatePenalties",
@@ -30,5 +31,6 @@
     "GetGrades",
     "ClearOutput",
     "LimitOutput",
-    "DeduplicateIds"
+    "DeduplicateIds",
+    "ClearHiddenTests"
 ]

nbgrader/preprocessors/clearhiddentests.py

@@ -0,0 +1,105 @@
+import re
+
+from traitlets import Bool, Unicode
+from textwrap import dedent
+
+from . import NbGraderPreprocessor
+from .. import utils
+
+
+class ClearHiddenTests(NbGraderPreprocessor):
+
+    begin_test_delimeter = Unicode(
+        "BEGIN HIDDEN TESTS",
+        help="The delimiter marking the beginning of hidden tests cases"
+    ).tag(config=True)
+
+    end_test_delimeter = Unicode(
+        "END HIDDEN TESTS",
+        help="The delimiter marking the end of hidden tests cases"
+    ).tag(config=True)
+
+    enforce_metadata = Bool(
+        True,
+        help=dedent(
+            """
+            Whether or not to complain if cells containing hidden test regions
+            are not marked as grade cells. WARNING: this will potentially cause
+            things to break if you are using the full nbgrader pipeline. ONLY
+            disable this option if you are only ever planning to use nbgrader
+            assign.
+            """
+        )
+    ).tag(config=True)
+
+    def _remove_hidden_test_region(self, cell):
+        """Find a region in the cell that is delimeted by
+        `self.begin_test_delimeter` and `self.end_test_delimeter` (e.g.  ###
+        BEGIN HIDDEN TESTS and ### END HIDDEN TESTS). Remove that region
+        depending the cell type.
+
+        This modifies the cell in place, and then returns True if a
+        hidden test region was removed, and False otherwise.
+        """
+        # pull out the cell input/source
+        lines = cell.source.split("\n")
+
+        new_lines = []
+        in_test = False
+        removed_test = False
+
+        for line in lines:
+            # begin the test area
+            if self.begin_test_delimeter in line:
+
+                # check to make sure this isn't a nested BEGIN HIDDEN TESTS
+                # region
+                if in_test:
+                    raise RuntimeError(
+                        "Encountered nested begin hidden tests statements")
+                in_test = True
+                removed_test = True
+
+            # end the solution area
+            elif self.end_test_delimeter in line:
+                in_test = False
+
+            # add lines as long as it's not in the hidden tests region
+            elif not in_test:
+                new_lines.append(line)
+
+        # we finished going through all the lines, but didn't find a
+        # matching END HIDDEN TESTS statment
+        if in_test:
+            raise RuntimeError("No end hidden tests statement found")
+
+        # replace the cell source
+        cell.source = "\n".join(new_lines)
+
+        return removed_test
+
+    def preprocess(self, nb, resources):
+        nb, resources = super(ClearHiddenTests, self).preprocess(nb, resources)
+        if 'celltoolbar' in nb.metadata:
+            del nb.metadata['celltoolbar']
+        return nb, resources
+
+    def preprocess_cell(self, cell, resources, cell_index):
+        # remove hidden test regions
+        removed_test = self._remove_hidden_test_region(cell)
+
+        # determine whether the cell is a grade cell
+        is_grade = utils.is_grade(cell)
+
+        # check that it is marked as a grade cell if we remove a test
+        # region -- if it's not, then this is a problem, because the cell needs
+        # to be given an id
+        if not is_grade and removed_test:
+            if self.enforce_metadata:
+                raise RuntimeError(
+                    "Hidden test region detected in a non-grade cell; "
+                    "please make sure all solution regions are within "
+                    "'Autograder tests' cells."
+                )
+
+        return cell, resources