From 4be09c834798d296c51d42111ac09f3f27382c5d Mon Sep 17 00:00:00 2001 From: ifaint Date: Wed, 20 May 2020 13:46:29 +0800 Subject: [PATCH 1/2] Added JupterNotebookUploadQuestion. --- course/page/__init__.py | 4 +- course/page/upload.py | 346 +++++++++++++----- .../file-upload-form-with-ipynb-preview.html | 17 + course/templates/course/file-upload-form.html | 38 +- course/utils.py | 157 ++++---- requirements.txt | 2 + tests/base_test_mixins.py | 4 +- tests/constants.py | 4 + tests/resource/test_notebook.ipynb | 105 ++++++ tests/test_pages/test_base.py | 8 +- tests/test_pages/test_generic.py | 33 ++ tests/test_pages/test_upload.py | 38 ++ tests/test_utils.py | 39 +- 13 files changed, 622 insertions(+), 173 deletions(-) create mode 100644 course/templates/course/file-upload-form-with-ipynb-preview.html create mode 100644 tests/resource/test_notebook.ipynb diff --git a/course/page/__init__.py b/course/page/__init__.py index e268f9d69..4e8cc1b62 100644 --- a/course/page/__init__.py +++ b/course/page/__init__.py @@ -37,7 +37,7 @@ ChoiceQuestion, MultipleChoiceQuestion, SurveyChoiceQuestion) from course.page.code import ( PythonCodeQuestion, PythonCodeQuestionWithHumanTextFeedback) -from course.page.upload import FileUploadQuestion +from course.page.upload import FileUploadQuestion, JupyterNotebookUploadQuestion __all__ = ( "InvalidPageData", @@ -51,7 +51,7 @@ "ChoiceQuestion", "SurveyChoiceQuestion", "MultipleChoiceQuestion", "PythonCodeQuestion", "PythonCodeQuestionWithHumanTextFeedback", - "FileUploadQuestion", + "FileUploadQuestion", "JupyterNotebookUploadQuestion", ) __doc__ = """ diff --git a/course/page/upload.py b/course/page/upload.py index cb2618de8..53a4887d4 100644 --- a/course/page/upload.py +++ b/course/page/upload.py @@ -2,7 +2,10 @@ from __future__ import division -__copyright__ = "Copyright (C) 2014 Andreas Kloeckner" +__copyright__ = """ +Copyright (C) 2014 Andreas Kloeckner +Copyright (c) 2020 Dong Zhuang +""" __license__ = """ Permission is hereby granted, free of charge, to any person obtaining a copy @@ -38,6 +41,13 @@ from crispy_forms.layout import Layout, Field +# {{{ mypy + +if False: + from typing import Optional, Text # noqa + +# }}} + # {{{ upload question @@ -46,11 +56,16 @@ class FileUploadForm(StyledForm): uploaded_file = forms.FileField(required=True, label=ugettext_lazy('Uploaded file')) - def __init__(self, maximum_megabytes, mime_types, *args, **kwargs): + def __init__(self, maximum_megabytes, mime_types, + clean_uploaded_file_callback=None, + *args, **kwargs): super(FileUploadForm, self).__init__(*args, **kwargs) self.max_file_size = maximum_megabytes * 1024**2 self.mime_types = mime_types + self.clean_uploaded_file_callback = clean_uploaded_file_callback + if self.clean_uploaded_file_callback is not None: + assert callable(self.clean_uploaded_file_callback) # 'accept=' doesn't work right for at least application/octet-stream. # We'll start with a whitelist. @@ -76,15 +91,151 @@ def clean_uploaded_file(self): % {'allowedsize': filesizeformat(self.max_file_size), 'uploadedsize': filesizeformat(uploaded_file.size)}) - if self.mime_types is not None and self.mime_types == ["application/pdf"]: - if uploaded_file.read()[:4] != b"%PDF": - raise forms.ValidationError(_("Uploaded file is not a PDF.")) + if self.clean_uploaded_file_callback: + uploaded_file = self.clean_uploaded_file_callback(uploaded_file) + + # Ensure the callback returns the uploaded file. + assert uploaded_file is not None, \ + "The callback should return the uploaded_file" return uploaded_file -class FileUploadQuestion(PageBaseWithTitle, PageBaseWithValue, +class FileUploadQuestionBase(PageBaseWithTitle, PageBaseWithValue, PageBaseWithHumanTextFeedback, PageBaseWithCorrectAnswer): + + @property + def ALLOWED_MIME_TYPES(self): # noqa + raise NotImplementedError + + @property + def file_extension(self): + return None + + form_template = "course/file-upload-form.html" + default_download_name = None # type: Optional[Text] + + def __init__(self, vctx, location, page_desc): + super(FileUploadQuestionBase, self).__init__(vctx, location, page_desc) + + if page_desc.maximum_megabytes <= 0: + raise ValidationError( + string_concat( + location, ": ", + _("'maximum_megabytes' expects a positive value, " + "got %(value)s instead") + % {'value': str(page_desc.maximum_megabytes)})) + + if vctx is not None: + if not hasattr(page_desc, "value"): + vctx.add_warning(location, _("upload question does not have " + "assigned point value")) + + self.mime_types = self.ALLOWED_MIME_TYPES + if hasattr(self.page_desc, "mime_types"): + self.mime_types = self.page_desc.mime_types + + def required_attrs(self): + return super(FileUploadQuestionBase, self).required_attrs() + ( + ("prompt", "markup"), + ("maximum_megabytes", (int, float)), + ) + + def allowed_attrs(self): + return super(FileUploadQuestionBase, self).allowed_attrs() + ( + ("correct_answer", "markup"), + ) + + def human_feedback_point_value(self, page_context, page_data): + return self.max_points(page_data) + + def markup_body_for_title(self): + return self.page_desc.prompt + + def body(self, page_context, page_data): + return markup_to_html(page_context, self.page_desc.prompt) + + @staticmethod + def _get_uploaded_file_buf(files_data): + files_data["uploaded_file"].seek(0) + return files_data["uploaded_file"].read() + + def files_data_to_answer_data(self, files_data): + buf = self._get_uploaded_file_buf(files_data) + + if len(self.mime_types) == 1: + mime_type, = self.mime_types + else: + mime_type = files_data["uploaded_file"].content_type + from base64 import b64encode + return { + "base64_data": b64encode(buf).decode(), + "mime_type": mime_type, + } + + def make_form(self, page_context, page_data, + answer_data, page_behavior): + form = FileUploadForm( + self.page_desc.maximum_megabytes, self.mime_types) + return form + + def process_form_post(self, page_context, page_data, post_data, files_data, + page_behavior): + form = FileUploadForm( + self.page_desc.maximum_megabytes, self.mime_types, + self.form_clean_uploaded_file_callback, + post_data, files_data) + return form + + def form_clean_uploaded_file_callback(self, uploaded_file): + """ This is used to handle uploaded_file field clean. + """ + return uploaded_file + + def get_form_to_html_context(self, request, page_context, form, answer_data): + ctx = {"form": form} + if answer_data is not None: + ctx["mime_type"] = answer_data["mime_type"] + ctx["data_url"] = "data:%s;base64,%s" % ( + answer_data["mime_type"], + answer_data["base64_data"], + ) + ctx["default_download_name"] = self.default_download_name + return ctx + + def form_to_html(self, request, page_context, form, answer_data): + ctx = self.get_form_to_html_context(request, page_context, form, answer_data) + from django.template.loader import render_to_string + return render_to_string(self.form_template, ctx, request) + + def answer_data(self, page_context, page_data, form, files_data): + return self.files_data_to_answer_data(files_data) + + def get_download_file_extension(self): + if self.file_extension is not None: + return self.file_extension + + ext = None + if len(self.mime_types) == 1: + mtype, = self.mime_types + from mimetypes import guess_extension + ext = guess_extension(mtype) + + if ext is None: + ext = ".dat" + return ext + + def normalized_bytes_answer(self, page_context, page_data, answer_data): + if answer_data is None: + return None + + ext = self.get_download_file_extension() + + from base64 import b64decode + return ext, b64decode(answer_data["base64_data"]) + + +class FileUploadQuestion(FileUploadQuestionBase): """ A page allowing the submission of a file upload that will be graded with text feedback by a human grader. @@ -175,100 +326,131 @@ def __init__(self, vctx, location, page_desc): set(page_desc.mime_types) - set(self.ALLOWED_MIME_TYPES))}) - if page_desc.maximum_megabytes <= 0: - raise ValidationError( - string_concat( - location, ": ", - _("'maximum_megabytes' expects a positive value, " - "got %(value)s instead") - % {'value': str(page_desc.maximum_megabytes)})) - - if vctx is not None: - if not hasattr(page_desc, "value"): - vctx.add_warning(location, _("upload question does not have " - "assigned point value")) - def required_attrs(self): return super(FileUploadQuestion, self).required_attrs() + ( - ("prompt", "markup"), - ("mime_types", list), - ("maximum_megabytes", (int, float)), - ) + ("mime_types", list),) - def allowed_attrs(self): - return super(FileUploadQuestion, self).allowed_attrs() + ( - ("correct_answer", "markup"), - ) + def form_clean_uploaded_file_callback(self, uploaded_file): + uploaded_file = super( + FileUploadQuestion, self + ).form_clean_uploaded_file_callback(uploaded_file) + if self.mime_types is not None and self.mime_types == ["application/pdf"]: + if uploaded_file.read()[:4] != b"%PDF": + raise forms.ValidationError(_("Uploaded file is not a PDF.")) + return uploaded_file - def human_feedback_point_value(self, page_context, page_data): - return self.max_points(page_data) - def markup_body_for_title(self): - return self.page_desc.prompt +# }}} - def body(self, page_context, page_data): - return markup_to_html(page_context, self.page_desc.prompt) - def files_data_to_answer_data(self, files_data): - files_data["uploaded_file"].seek(0) - buf = files_data["uploaded_file"].read() +class JupyterNotebookUploadQuestion(FileUploadQuestionBase): + """ + A page allowing the submission of a JupyterNotebook file that will be + graded with text feedback by a human grader. - if len(self.page_desc.mime_types) == 1: - mime_type, = self.page_desc.mime_types - else: - mime_type = files_data["uploaded_file"].content_type - from base64 import b64encode - return { - "base64_data": b64encode(buf).decode(), - "mime_type": mime_type, - } + .. attribute:: id - def make_form(self, page_context, page_data, - answer_data, page_behavior): - form = FileUploadForm( - self.page_desc.maximum_megabytes, self.page_desc.mime_types) - return form + |id-page-attr| - def process_form_post(self, page_context, page_data, post_data, files_data, - page_behavior): - form = FileUploadForm( - self.page_desc.maximum_megabytes, self.page_desc.mime_types, - post_data, files_data) - return form + .. attribute:: type - def form_to_html(self, request, page_context, form, answer_data): - ctx = {"form": form} - if answer_data is not None: - ctx["mime_type"] = answer_data["mime_type"] - ctx["data_url"] = "data:%s;base64,%s" % ( - answer_data["mime_type"], - answer_data["base64_data"], - ) + ``Page`` - from django.template.loader import render_to_string - return render_to_string( - "course/file-upload-form.html", ctx, request) + .. attribute:: is_optional_page - def answer_data(self, page_context, page_data, form, files_data): - return self.files_data_to_answer_data(files_data) + |is-optional-page-attr| - def normalized_bytes_answer(self, page_context, page_data, answer_data): - if answer_data is None: - return None + .. attribute:: access_rules - ext = None - if len(self.page_desc.mime_types) == 1: - mtype, = self.page_desc.mime_types - from mimetypes import guess_extension - ext = guess_extension(mtype) + |access-rules-page-attr| - if ext is None: - ext = ".dat" + .. attribute:: title - from base64 import b64decode - return (ext, b64decode(answer_data["base64_data"])) + |title-page-attr| -# }}} + .. attribute:: value + + |value-page-attr| + + .. attribute:: prompt + + Required. + The prompt for this question, in :ref:`markup`. + + .. attribute:: maximum_megabytes + + Required. + The largest file size + (in `Mebibyte `) + that the page will accept. + + .. attribute:: correct_answer + + Optional. + Content that is revealed when answers are visible + (see :ref:`flow-permissions`). Written in :ref:`markup`. + + .. attribute:: correct_answer + + Optional. + Content that is revealed when answers are visible + (see :ref:`flow-permissions`). Written in :ref:`markup`. + + .. attribute:: rubric + + Required. + The grading guideline for this question, in :ref:`markup`. + """ + + ALLOWED_MIME_TYPES = [ + "application/x-ipynb+json" + ] + file_extension = ".ipynb" + form_template = "course/file-upload-form-with-ipynb-preview.html" + default_download_name = "my_notebook.ipynb" + + def files_data_to_answer_data(self, files_data): + buf = self._get_uploaded_file_buf(files_data) + + from course.utils import render_notebook_from_source + from base64 import b64encode + return { + "base64_data": b64encode(buf).decode(), + "mime_type": self.mime_types[0], + "preview_base64_data": b64encode( + render_notebook_from_source(buf.decode()).encode()).decode() + } + + def get_form_to_html_context(self, request, page_context, form, answer_data): + ctx = super(JupyterNotebookUploadQuestion, self).get_form_to_html_context( + request, page_context, form, answer_data) + if answer_data is not None: + ctx["preview_data_url"] = "data:text/html;base64,%s" % ( + answer_data["preview_base64_data"], + ) + ctx["preview_base64_data"] = answer_data["preview_base64_data"] + + return ctx + + def form_clean_uploaded_file_callback(self, uploaded_file): + uploaded_file = super( + JupyterNotebookUploadQuestion, self + ).form_clean_uploaded_file_callback(uploaded_file) + import sys + from nbformat.reader import read + try: + if sys.version_info < (3, 6): + # nbformat.reader.read is assuming Python 3.6+ + import json + json.loads(uploaded_file.read().decode('utf-8')) + else: + read(uploaded_file) + except Exception as e: + tp, e, _ = sys.exc_info() + raise forms.ValidationError( + "%(err_type)s: %(err_str)s" + % {"err_type": tp.__name__, "err_str": str(e)}) + return uploaded_file # vim: foldmethod=marker diff --git a/course/templates/course/file-upload-form-with-ipynb-preview.html b/course/templates/course/file-upload-form-with-ipynb-preview.html new file mode 100644 index 000000000..755259825 --- /dev/null +++ b/course/templates/course/file-upload-form-with-ipynb-preview.html @@ -0,0 +1,17 @@ +{% extends "course/file-upload-form.html" %} +{% load i18n %} +{% block file_upload_embed_viewer_js %} + +{% endblock %} diff --git a/course/templates/course/file-upload-form.html b/course/templates/course/file-upload-form.html index 43d8717cc..d15387f6f 100644 --- a/course/templates/course/file-upload-form.html +++ b/course/templates/course/file-upload-form.html @@ -5,24 +5,32 @@
- {% trans "Review uploaded file" %} + {% trans "Review uploaded file" %} · - {% trans "Embed viewer" %} + {% trans "Preview" %}
- + {% endblock %} + +