diff --git a/cms/grading/Sandbox.py b/cms/grading/Sandbox.py index 9e8b5863d5..f40934bb0f 100644 --- a/cms/grading/Sandbox.py +++ b/cms/grading/Sandbox.py @@ -1420,7 +1420,7 @@ def initialize_isolate(self): + (["--cg"] if self.cgroup else []) + ["--box-id=%d" % self.box_id, "--init"]) try: - subprocess.check_call(init_cmd) + subprocess.check_call(init_cmd, stdout=subprocess.DEVNULL) except subprocess.CalledProcessError as e: raise SandboxInterfaceException( "Failed to initialize sandbox") from e diff --git a/cms/io/Repository.py b/cms/io/Repository.py index d02d557c16..ac994b07b6 100644 --- a/cms/io/Repository.py +++ b/cms/io/Repository.py @@ -41,10 +41,11 @@ class Repository: You have to use one repository object for all of these! """ - def __init__(self, path, auto_sync=False): + def __init__(self, path, auto_sync=False, auto_push=False): self.lock = Manager().Lock() self.path = path self.auto_sync = auto_sync + self.auto_push = auto_push def __enter__(self): self.lock.acquire() @@ -56,82 +57,101 @@ def __exit__(self, type, value, traceback): def _sync(self): if self.auto_sync: logger.info("Synchronizing {}".format(self.path)) - - with chdir(self.path): - gitout = "" - - try: - gitout = check_output(["git", "pull"]) - except: - logger.error("Couldn't sync with repository: " + - "{}".format(gitout)) - else: - logger.info("Finished synchronization: " + - "{}".format(gitout)) - - gitout = "" - + self._pull() + if self.auto_push: + self._push() + + def _pull(self): + logger.info("Pulling {}".format(self.path)) + + with chdir(self.path): + gitout = "" + + try: + gitout = check_output(["git", "pull"]) + except: + logger.error("Couldn't pull from repository " + + "({})".format(gitout)) + else: + logger.info("Finished pulling: " + + "{}".format(gitout)) + + def _push(self): + logger.info("Pushing {}".format(self.path)) + + with chdir(self.path): + gitout = "" + + try: + gitout = check_output(["git", "push"]) + except: + logger.error("Couldn't push to repository " + + "({})".format(gitout)) + else: + logger.info("Finished pushing: " + + "{}".format(gitout)) + + # For GerTranslate + # TODO Show errors in web overview + def commit(self, file_path, file_identifier): + # TODO Only do this if it's a git repository + # if self.auto_sync: + logger.info("Committing {} in {}".format(file_path, self.path)) + + with chdir(self.path): + gitout = "" + + try: + gitout = check_output(["git", "add", + file_path]) + except: + logger.error("Couldn't add file to git staging area: " + + "{}".format(gitout)) + else: try: - gitout = check_output(["git", "push"]) + gitout = "" + # NOTE file_path is relative to self.path, which isn't + # necessarily the root of the git repo. So the commit + # message might be confusing. + gitout = \ + check_output( + ["git", "commit", + "-o", file_path, + # TODO Provide meaningful commit message and + # author + "-m", "Changes to " + + file_identifier + + ", uploaded via GerTranslate web " + "interface", + "--author", '"GerTranslate "'] + ) except: - logger.error("Couldn't push to repository: " + + logger.error("Couldn't commit in repository: " + "{}".format(gitout)) else: - logger.info("Finished pushing: " + + logger.info("Committed: " + "{}".format(gitout)) - #For GerTranslate - #TODO Show errors in web overview - def commit(self, file_path, file_identifier): - #TODO Only do this if it's a git repository - #if self.auto_sync: - logger.info("Committing {} in {}".format(file_path,self.path)) - - with chdir(self.path): - gitout = "" - - try: - gitout = check_output(["git", "add", - file_path]) - except: - logger.error("Couldn't add file to git staging area: " + - "{}".format(gitout)) - else: - try: - gitout = "" - #NOTE file_path is relative to self.path, which isn't - #necessarily the root of the git repo. So the commit message - #might be confusing. - gitout = check_output(["git", "commit", - "-o", file_path, - "-m","Changes to "+file_identifier+", uploaded via GerTranslate web interface", - #TODO Provide meaningful commit message and author - "--author","\"GerTranslate \""]) - except: - logger.error("Couldn't commit in repository: " + - "{}".format(gitout)) - else: - logger.info("Committed: " + - "{}".format(gitout)) - - #For GerTranslate - #TODO Show errors in web overview + # For GerTranslate + # TODO Show errors in web overview def getlog(self, file_path): - #TODO Only do this if it's a git repository - #if self.auto_sync: - with chdir(self.path): - gitout = "" - - try: - #TODO Remove diff info lines - gitout = check_output(["git", "log", - '--pretty=format:Date: %ci%n%n %s%n', - "-p", - "--word-diff=color", - file_path]) - except: - logger.error("Couldn't get log: " + - "{}".format(gitout)) - else: - gitout = gitout.decode('utf-8') - return gitout + # TODO Only do this if it's a git repository + # if self.auto_sync: + with chdir(self.path): + gitout = "" + + try: + # TODO Remove diff info lines + gitout = check_output( + ["git", "log", + '--pretty=format:Date: %ci%n%n %s%n', + "-p", + "--word-diff=color", + file_path] + ) + except: + logger.error("Couldn't get log: " + + "{}".format(gitout)) + else: + gitout = gitout.decode('utf-8') + return gitout diff --git a/cms/io/TaskAccess.py b/cms/io/TaskAccess.py index 06e38db185..af024ff970 100644 --- a/cms/io/TaskAccess.py +++ b/cms/io/TaskAccess.py @@ -43,11 +43,12 @@ logger = logging.getLogger(__name__) + def unpack_code(code): - if code.count("/")>=2: - contest,task,language = code.split("/") + if code.count("/") >= 2: + contest, task, language = code.split("/") else: - contest,task,language = [""] + code.split("/") + contest, task, language = [""] + code.split("/") if contest != "" and contest not in TaskTranslateInfo.contests: raise KeyError("No such contest") @@ -56,23 +57,26 @@ def unpack_code(code): if language not in TaskTranslateInfo.languages and language != "ALL": raise KeyError("No such language") - return contest,task,language + return contest, task, language + -def repository_code(contest,task,language): +def repository_code(contest, task, language): srcname = TaskTranslateInfo.tasks[task]["filename"] - p = Path(contest,task) if contest else Path(task) + p = Path(contest, task) if contest else Path(task) if language is not None: return p / (srcname+"-"+language+".tex") else: return p / srcname+".tex" -def repository_lock_file_code(contest,task,language): - p = Path(contest,task) if contest else Path(task) + +def repository_lock_file_code(contest, task, language): + p = Path(contest, task) if contest else Path(task) if language is not None: return p / (language+".lock") else: return p / "O.lock" + class TaskCompileJob: def __init__(self, repository, contest, name, balancer, language=None): self.repository = repository @@ -120,7 +124,9 @@ def do(status, repository, balancer): try: i = json.loads(lan_path.open().read()) except: - i = {"error": "The languages.json file is corrupt."} + i = { + "error": "The languages.json file is corrupt." + } else: missing = [] for e in ["languages"]: @@ -128,26 +134,31 @@ def do(status, repository, balancer): missing.append(e) if len(missing) > 0: - i["error"] = "Some important entries are missing: " + \ - ", ".join(missing) + "." + i["error"] = \ + "Some important entries are missing: " + \ + ", ".join(missing) + "." - #TODO do this right, i.e.: Why copyifnecessary? + # TODO do this right, i.e.: Why copyifnecessary? if is_overview: copyifnecessary(lan_path, - Path(self.repository.path) / - self.contest / - "languages.json" - ) + Path(self.repository.path) / + self.contest / + "languages.json" + ) - comp = GerMake(repository.path + "/" + self.contest, - task=self.name if not is_overview else "NO_TASK", - no_test=True, - submission=None, - no_latex=False, - safe_latex=True, - language=self.language, - clean=False, - minimal=True) + comp = \ + GerMake( + repository.path + "/" + self.contest, + task=self.name if not is_overview else "NO_TASK", + no_test=True, + submission=None, + no_latex=False, + verbose_latex=True, + safe_latex=True, + language=self.language, + clean=False, + minimal=True + ) with repository: comp.prepare() @@ -168,30 +179,33 @@ def do(status, repository, balancer): else: languages = [self.language] print(languages) + def f(contestconfig): for l in languages: contestconfig.user( "[user-{}]".format(l), "[password]", "Jane", "Doe", - group=MyGroup("dummy", 0, 0, 0, 0, None), + group=MyGroup( + "dummy", 0, 0, 0, 0, None), primary_statements=[l] - ) + ) - #If self.language is None, this is the primary statement. - #If self.language is ALL, this is _a list_ of all statements. - #Else, it's the task statement associated with that language. + # If self.language is None, this is the primary statement. + # If self.language is ALL, this is _a list_ of all statements. + # Else, it's the task statement associated with that language. pdf_file = comp.build(extra_conf_f=f) if is_overview: - filename = "overview-sheets-for-" + self.language + ".pdf" + filename = \ + "overview-sheets-for-{}.pdf".format(self.language) pdf_file = str(Path(self.repository.path) / - self.contest / - "build" / - "overview" / - ".overviews-per-language" / - filename - ) + self.contest / + "build" / + "overview" / + ".overviews-per-language" / + filename + ) if pdf_file is None: status["error"] = True @@ -205,11 +219,11 @@ def f(contestconfig): for s in pdf_file: pdfmerger.append(s) pdf_file = str(Path(self.repository.path) / - self.contest / - "build" / - self.name / - "statement-ALL.pdf" - ) + self.contest / + "build" / + self.name / + "statement-ALL.pdf" + ) pdfmerger.write(pdf_file) pdfmerger.close() @@ -295,18 +309,18 @@ def __init__(self, repository, name): self.name = name def get(self): - result = None#TODO + result = None # TODO - contest,task,language = unpack_code(self.name) - _repository_code = repository_code(contest,task,language) + contest, task, language = unpack_code(self.name) + _repository_code = repository_code(contest, task, language) tex_file = Path(self.repository.path) / _repository_code logger.info(str(_repository_code) + " is accessed.") if tex_file is None: - #TODO Handle the error (you should probably implement a info function - #like above, then implement a query function in TaskAccess - #and handle the result in download.js + # TODO Handle the error (you should probably implement an info + # function like above, then implement a query function in + # TaskAccess and handle the result in download.js error = "No statement TeX found" else: @@ -324,31 +338,33 @@ def __init__(self, repository, name): def receive(self, f): result = None - contest,task,language = unpack_code(self.name) - _repository_code = repository_code(contest,task,language) + contest, task, language = unpack_code(self.name) + _repository_code = repository_code(contest, task, language) tex_file = Path(self.repository.path) / _repository_code - _repository_lock_file_code = repository_lock_file_code(contest,task,language) + _repository_lock_file_code = repository_lock_file_code( + contest, task, language) lock_file = Path(self.repository.path) / _repository_lock_file_code if lock_file.is_file(): - #TODO Handle error + # TODO Handle error error = "Translation already locked; currently, you can't change"\ - "this translation. Please contact an administrator." + "this translation. Please contact an administrator." else: logger.info(str(_repository_code) + " is written to.") if f is None: - #TODO Handle the error (via result?) + # TODO Handle the error (via result?) error = "No file received" - elif len(f)>1048576: #1MB + elif len(f) > 1048576: # 1MB error = "File too big" else: with open(tex_file, "wb") as target_file: target_file.write(f) - self.repository.commit(str(tex_file.resolve()), str(_repository_code)) + self.repository.commit( + str(tex_file.resolve()), str(_repository_code)) return result @@ -359,15 +375,17 @@ def __init__(self, repository, name): self.name = name def mark(self): - contest,task,language = unpack_code(self.name) - _repository_lock_file_code = repository_lock_file_code(contest,task,language) + contest, task, language = unpack_code(self.name) + _repository_lock_file_code = repository_lock_file_code( + contest, task, language) lock_file = Path(self.repository.path) / _repository_lock_file_code logger.info(str(_repository_lock_file_code) + " is created.") with open(lock_file, "w") as target_file: target_file.write("The translation in this language is locked.") - self.repository.commit(str(lock_file.resolve()), str(_repository_lock_file_code)) + self.repository.commit(str(lock_file.resolve()), + str(_repository_lock_file_code)) class TaskGitLog: @@ -376,18 +394,18 @@ def __init__(self, repository, name): self.name = name def get(self): - result = None#TODO + result = None # TODO - contest,task,language = unpack_code(self.name) - _repository_code = repository_code(contest,task,language) + contest, task, language = unpack_code(self.name) + _repository_code = repository_code(contest, task, language) tex_file = Path(self.repository.path) / _repository_code logger.info("Git log for " + str(_repository_code) + " is accessed.") if tex_file is None: - #TODO Handle the error (you should probably implement a info function - #like above, then implement a query function in TaskAccess - #and handle the result in download.js + # TODO Handle the error (you should probably implement an info + # function like above, then implement a query function in + # TaskAccess and handle the result in download.js error = "No statement TeX found" else: @@ -411,11 +429,14 @@ def init(repository, max_compilations): @staticmethod def compile(name): - contest,task,language = unpack_code(name) + contest, task, language = unpack_code(name) if name not in TaskAccess.jobs: - TaskAccess.jobs[name] = TaskCompileJob(TaskAccess.repository, contest, task, - TaskAccess.balancer, language) + TaskAccess.jobs[name] = TaskCompileJob(TaskAccess.repository, + contest, + task, + TaskAccess.balancer, + language) return TaskAccess.jobs[name].join() @staticmethod @@ -449,4 +470,6 @@ def mark(name): @staticmethod def getGitLog(name): C = Ansi2HTMLConverter() - return {"log": C.convert(TaskGitLog(TaskAccess.repository, name).get())} + return { + "log": C.convert(TaskGitLog(TaskAccess.repository, name).get()) + } diff --git a/cms/io/TaskFetch.py b/cms/io/TaskFetch.py index 5699ad837a..0d67fbc449 100644 --- a/cms/io/TaskFetch.py +++ b/cms/io/TaskFetch.py @@ -74,7 +74,7 @@ def do(status, repository, balancer): with balancer: try: comp = GerMakeTask(repository.path, self.name, True, True, - None, False, None, False) + None, False, True, None, False) with repository: comp.prepare() diff --git a/cms/server/gertranslate/server.py b/cms/server/gertranslate/server.py index ec8162b525..543e656177 100644 --- a/cms/server/gertranslate/server.py +++ b/cms/server/gertranslate/server.py @@ -46,7 +46,7 @@ def get(self): class TaskCompileHandler(RequestHandler): def get(self): self.write(TaskAccess.query(self.get_argument("code"), - int(self.get_argument("handle")))) + int(self.get_argument("handle")))) self.flush() def post(self): @@ -57,12 +57,15 @@ def post(self): class PDFHandler(RequestHandler): def share(self, statement, code): self.set_header("Content-Type", "application/pdf") - #TODO Do this less ugly. + # TODO Do this less ugly. srcname = TaskTranslateInfo.tasks[unpack_code(code)[1]]["filename"] prefix = "statement-" if srcname == "statement" else "" self.set_header( "Content-Disposition", - "attachment;filename=\"{}{}.pdf\"".format(prefix,code.replace('/','-'))) + "attachment;filename=\"{}{}.pdf\"".format( + prefix, code.replace('/', '-') + ) + ) self.write(statement) self.flush() @@ -82,16 +85,18 @@ def get(self, code): class TeXHandler(RequestHandler): def share(self, statement, code): self.set_header("Content-Type", "text") - #TODO Do this less ugly. + # TODO Do this less ugly. srcname = TaskTranslateInfo.tasks[unpack_code(code)[1]]["filename"] - code = code[1:] if code[0]=='/' else code + code = code[1:] if code[0] == '/' else code if srcname == "statement": srcname += "-" else: srcname = "" self.set_header( "Content-Disposition", - "attachment;filename=\""+srcname+"{}.tex\"".format(code.replace('/','-'))) + "attachment;filename=\"" + srcname + + "{}.tex\"".format(code.replace('/', '-')) + ) self.write(statement) self.flush() @@ -102,16 +107,17 @@ def get(self, code): if statement is None: raise ValueError except: - logger.error("could not download statement TeX file for {}".format(code)) - self.render("error.html")#TODO + logger.error( + "could not download statement TeX file for {}".format(code)) + self.render("error.html") # TODO else: self.share(statement, code) class UploadHandler(RequestHandler): def post(self, code): - #TODO Handle Error - #TODO Check file size + # TODO Handle Error + # TODO Check file size f = self.request.files['file'][0]['body'] TaskAccess.receiveTeX(code, f) @@ -142,8 +148,9 @@ def get(self, code): if gitlog is None: raise ValueError except: - logger.error("could not download statement log for {}".format(code)) - self.render("error.html")#TODO + logger.error( + "could not download statement log for {}".format(code)) + self.render("error.html") # TODO else: self.write(gitlog) self.flush() @@ -152,9 +159,9 @@ def get(self, code): class GerTranslateWebServer: """Service running a web server that lets you download task statements and upload translations - For a future implementation, there should be something like an nginx configuration with one user - per language, where all users have access to /, but /it/ is restricted - to user 'it'. + For a future implementation, there should be something like an nginx + configuration with one user per language, where all users have access + to /, but /it/ is restricted to user 'it'. """ def __init__(self): @@ -173,7 +180,9 @@ def __init__(self): "static_path": resource_filename("cms.server", "gertranslate/static")} - repository = Repository(config.translate_task_repository, config.translate_auto_sync) + repository = Repository(config.translate_task_repository, + config.translate_auto_sync, + auto_push=True) TaskAccess.init(repository, config.translate_max_compilations) TaskTranslateInfo.init(repository) diff --git a/cmscontrib/AddParticipation.py b/cmscontrib/AddParticipation.py index 1ff146e772..422925fec5 100755 --- a/cmscontrib/AddParticipation.py +++ b/cmscontrib/AddParticipation.py @@ -48,7 +48,7 @@ def add_participation(username, contest_id, ip, delay_time, extra_time, password, method, is_hashed, team_code, hidden, - unrestricted, groupname): + unofficial, unrestricted, groupname): logger.info("Creating the user's participation in the database.") delay_time = delay_time if delay_time is not None else 0 extra_time = extra_time if extra_time is not None else 0 diff --git a/cmstestsuite/unit_tests/cmscontrib/AddParticipationTest.py b/cmstestsuite/unit_tests/cmscontrib/AddParticipationTest.py index 924ba8e046..001eec5f21 100755 --- a/cmstestsuite/unit_tests/cmscontrib/AddParticipationTest.py +++ b/cmstestsuite/unit_tests/cmscontrib/AddParticipationTest.py @@ -67,48 +67,48 @@ def assertParticipationInDb(self, user_id, contest_id, password, def test_success(self): self.assertTrue(add_participation( self.user.username, self.contest.id, None, None, None, - "pwd", "bcrypt", False, None, False, False, + "pwd", "bcrypt", False, None, False, True, False, self.contest.main_group.name)) self.assertParticipationInDb(self.user.id, self.contest.id, "pwd") def test_success_with_team(self): self.assertTrue(add_participation( self.user.username, self.contest.id, None, None, None, - "pwd", "bcrypt", False, self.team.code, False, False, None)) + "pwd", "bcrypt", False, self.team.code, False, True, False, None)) self.assertParticipationInDb(self.user.id, self.contest.id, "pwd", team_code=self.team.code) def test_user_not_found(self): self.assertFalse(add_participation( self.user.username + "_", self.contest.id, None, None, None, - "pwd", "bcrypt", False, None, False, False, None)) + "pwd", "bcrypt", False, None, False, True, False, None)) def test_contest_not_found(self): self.assertFalse(add_participation( self.user.username, self.contest.id + 1, None, None, None, - "pwd", "bcrypt", False, None, False, False, None)) + "pwd", "bcrypt", False, None, False, True, False, None)) def test_team_not_found(self): self.assertFalse(add_participation( self.user.username, self.contest.id, None, None, None, - "pwd", "bcrypt", False, self.team.code + "_", False, False, None)) + "pwd", "bcrypt", False, self.team.code + "_", False, True, False, None)) def test_already_exists(self): self.assertTrue(add_participation( self.user.username, self.contest.id, None, None, None, - "pwd", "bcrypt", False, None, False, False, None)) + "pwd", "bcrypt", False, None, False, True, False, None)) self.assertParticipationInDb(self.user.id, self.contest.id, "pwd") # Second add_participation should fail without changing values. self.assertFalse(add_participation( self.user.username, self.contest.id + 1, "1.2.3.4", 60, 120, - "other_pwd", "plaintext", True, self.team.code, True, True, None)) + "other_pwd", "plaintext", True, self.team.code, True, False, True, None)) self.assertParticipationInDb(self.user.id, self.contest.id, "pwd") def test_other_values(self): self.assertTrue(add_participation( self.user.username, self.contest.id, "1.2.3.4", 60, 120, - "pwd", "plaintext", True, None, True, True, None)) + "pwd", "plaintext", True, None, True, False, True, None)) self.assertParticipationInDb(self.user.id, self.contest.id, "pwd", delay_time=60, extra_time=120, hidden=True, unrestricted=True, diff --git a/cmstestsuite/unit_tests/server/contest/submission/workflow_test.py b/cmstestsuite/unit_tests/server/contest/submission/workflow_test.py index ffb93ac017..0b2bd3f959 100755 --- a/cmstestsuite/unit_tests/server/contest/submission/workflow_test.py +++ b/cmstestsuite/unit_tests/server/contest/submission/workflow_test.py @@ -153,7 +153,7 @@ def call(self): return accept_submission( self.session, self.file_cacher, self.participation, self.task, self.timestamp, self.tornado_files, self.language_name, - self.official) + self.official, True) def assertSubmissionIsValid(self, submission, timestamp, language, files, official): diff --git a/requirements.txt b/requirements.txt index 08e99f81b9..04c0e64b7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,8 +9,9 @@ psutil>=5.5,<5.6 # https://github.com/giampaolo/psutil/blob/master/HISTORY.rst requests>=2.22,<2.23 # https://pypi.python.org/pypi/requests # Slightly higher version for Python 3.8 support # gevent>=1.5,<1.6 # http://www.gevent.org/changelog.html -# We override this for Python 3.9 support -gevent>=1.5 # http://www.gevent.org/changelog.html +# We override this for Python 3.9 support: +# 20.6.0 introduced Python 3.9 support; 20.9 would depend on greenlet>=0.4.17. +gevent>=20.6.0,<20.9 # http://www.gevent.org/changelog.html # Limit greenlet version for binary compatibility with gevent 1.5 wheels greenlet<0.4.17 werkzeug>=0.16,<0.17 # https://github.com/pallets/werkzeug/blob/master/CHANGES