From 4d55334d65972f03936ca6eec185bd25313e4a19 Mon Sep 17 00:00:00 2001 From: DaveWJRH Date: Mon, 29 Jun 2026 20:01:45 -0400 Subject: [PATCH 1/2] Added docker compose file, Judge0 configuration file, a simple python tester script, and a few markdown files for documentation purposes --- Languages.md | 101 ++++++++ README.md | 114 +++++---- Statuses.md | 18 ++ docker-compose.yml | 47 ++++ judge0.conf | 358 +++++++++++++++++++++++++++++ questions/add-two-numbers/tests.py | 11 +- test_judge0.py | 107 +++++++++ 7 files changed, 711 insertions(+), 45 deletions(-) create mode 100644 Languages.md create mode 100644 Statuses.md create mode 100644 docker-compose.yml create mode 100644 judge0.conf create mode 100644 test_judge0.py diff --git a/Languages.md b/Languages.md new file mode 100644 index 0000000..22714d7 --- /dev/null +++ b/Languages.md @@ -0,0 +1,101 @@ +# Judge0 + +## Supported/Active Languages + +### By Name +* Assembly (NASM 2.14.02) +* Bash (5.0.0) +* Basic (FBC 1.07.1) +* C (Clang 7.0.1) +* C++ (Clang 7.0.1) +* C (GCC 7.4.0) +* C++ (GCC 7.4.0) +* C (GCC 8.3.0) +* C++ (GCC 8.3.0) +* C (GCC 9.2.0) +* C++ (GCC 9.2.0) +* Clojure (1.10.1) +* C# (Mono 6.6.0.161) +* COBOL (GnuCOBOL 2.2) +* Common Lisp (SBCL 2.0.0) +* D (DMD 2.089.1) +* Elixir (1.9.4) +* Erlang (OTP 22.2) +* Executable +* F# (.NET Core SDK 3.1.202) +* Fortran (GFortran 9.2.0) +* Go (1.13.5) +* Groovy (3.0.3) +* Haskell (GHC 8.8.1) +* Java (OpenJDK 13.0.1) +* JavaScript (Node.js 12.14.0) +* Kotlin (1.3.70) +* Lua (5.3.5) +* Multi-file program +* Objective-C (Clang 7.0.1) +* OCaml (4.09.0) +* Octave (5.1.0) +* Pascal (FPC 3.0.4) +* Perl (5.28.1) +* PHP (7.4.1) +* Plain Text +* Prolog (GNU Prolog 1.4.5) +* Python (2.7.17) +* Python (3.8.1) +* R (4.0.0) +* Ruby (2.7.0) +* Rust (1.40.0) +* Scala (2.13.2) +* SQL (SQLite 3.27.2) +* Swift (5.2.3) +* TypeScript (3.7.4) +* Visual Basic.Net (vbnc 0.0.0.5943) + +### By ID +43. Plain Text +44. Executable +45. Assembly (NASM 2.14.02) +46. Bash (5.0.0) +47. Basic (FBC 1.07.1) +48. C (GCC 7.4.0) +49. C (GCC 8.3.0) +50. "C (GCC 9.2.0) +51. "C# (Mono 6.6.0.161) +52. "C++ (GCC 7.4.0) +53. C++ (GCC 8.3.0) +54. C++ (GCC 9.2.0) +55. Common Lisp (SBCL 2.0.0) +56. D (DMD 2.089.1) +57. Elixir (1.9.4) +58. Erlang (OTP 22.2) +59. Fortran (GFortran 9.2.0) +60. Go (1.13.5) +61. Haskell (GHC 8.8.1) +62. Java (OpenJDK 13.0.1) +63. JavaScript (Node.js 12.14.0) +64. Lua (5.3.5) +65. OCaml (4.09.0) +66. Octave (5.1.0) +67. Pascal (FPC 3.0.4) +68. PHP (7.4.1) +69. Prolog (GNU Prolog 1.4.5) +70. Python (2.7.17) +71. Python (3.8.1) +72. Ruby (2.7.0) +73. Rust (1.40.0) +74. TypeScript (3.7.4) +75. C (Clang 7.0.1) +76. C++ (Clang 7.0.1) +77. COBOL (GnuCOBOL 2.2) +78. Kotlin (1.3.70) +79. Objective-C (Clang 7.0.1) +80. R (4.0.0) +81. Scala (2.13.2) +82. SQL (SQLite 3.27.2) +83. Swift (5.2.3) +84. Visual Basic.Net (vbnc 0.0.0.5943) +85. Perl (5.28.1) +86. Clojure (1.10.1) +87. F# (.NET Core SDK 3.1.202) +88. Groovy (3.0.3) +89. Multi-file program diff --git a/README.md b/README.md index 20d73ab..ae32b72 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,77 @@ -# Code Execution Question Spike - -## What this repo is - -This is a time-boxed investigation prototype for adding code-execution question support to [questions.carletoncomputerscience.ca](https://questions.carletoncomputerscience.ca/). It is not production code and will not be deployed. - -Three approaches are being evaluated in parallel via spike tickets. Once viable options are identified, the chosen approach is migrated into the main Astro repo and this repo is archived. Nothing here is intended to be maintained long-term. - -## Approaches under evaluation - -### (a) Pyodide — client-side Python execution - -Pyodide runs CPython compiled to WebAssembly, executing Python entirely in the browser with no server involvement. The spike evaluates whether student-submitted Python functions can be loaded, executed against a test suite, and graded without any network round-trip. Key unknowns: bundle size, startup latency, and sandbox adequacy for an exam-adjacent context. - -### (b) CheerpJ (or similar) — client-side Java execution - -CheerpJ compiles Java bytecode to WebAssembly/JavaScript, allowing Java programs to run in-browser. The spike evaluates whether student Java submissions can be compiled or pre-compiled and executed client-side with comparable grading semantics to the Python approach. Key unknowns: whether student code can be compiled in-browser or requires a pre-compilation step, runtime overhead, and JVM subset limitations. -Other options may include TeaVM, QEMU-WASM, or something else. - -### (c) Judge0 — server-side execution backend - -Judge0 is an open-source code execution API that runs submissions in isolated containers server-side. The spike evaluates the integration complexity of setting up [Judge0](https://judge0.com/) (self-hosted), the latency and infrastructure overhead, the process of configuring questions on the platform, and the viability as a fallback for languages that cannot run client-side. - ---- - -The goal is to have either a way to mark both Python and Java questions fully locally in the user's browser, and/or to determine the Judge0 integration process and complexity. Python and Java are the two priority languages as they are used in the intro courses. +# Judge0 +API: **https://ce.judge0.com/** + +## Goals +* Self-host a Judge0 instance. +* Submit a student's Python or Java submission alongside a test file. +* Get back pass/fail results matching the JSON format in `SCHEMA.md`. +* Understand what it takes to keep this running reliably. + +## Findings + +1. **Setup complexity.** + What does self-hosting Judge0 actually involve? Walk through what you did: prerequisites, how long it took, and what was confusing or poorly documented. Also note whether the public hosted API is a realistic alternative — cover any rate limits, cost, or data concerns. + +2. **Infrastructure requirements.** +What kind of server does Judge0 need to run comfortably? What happens during a busy period — say, 100 students submitting simultaneously during a lab? What does it mean for a volunteer society to own and operate this long-term: keeping it running, handling updates, dealing with abuse, cost of hosting? + +***Docker Statistics*** +- Container CPU Usage: 12%/1200% (12 CPUs available) +With 20 runs, 200%/1200% +- Container memory usage: 2.28GB/15.17GB +- Compose stack: + - server (judge0/judge0:1.13.1) + - workers (judge0/judge0:1.13.1) + *\* Additional worker nodes can be deployed to process code execution in parallel during traffic spikes* + - db (postgres:16.2) + *\* Stores persistent configuration, submission records, and language metadata.* + - redis (redis:7.2.4) + +3. **End-to-end latency.** +From submission sent to result received, how long does it take? Break it down: queue wait, execution time, API round-trip. What's the experience like for a student waiting on feedback? + +4. **Question and submission workflow.** +A test file and the corresponding submission code can be appended together to execute the test suite using the user provided solution. The 200 OK Response will be JSON formatted to include: +``` +{ + "stdout": [Standard output of the program after execution], + "time": [Program’s run time], + "memory": [Memory used by the program after execution], + "stderr": [Standard error of the program after execution], + "token": [Unique submission token which can be used to get a specific submission], + "compile_output:" [Compiler output after compilation], + "message": [If submission status is Internal Error then this message comes from Judge0 itself, otherwise this is status message from isolate], + "status": [Submission status] + { + "id": [The status code as listed in Statuses.md], + "description":[A description for the corresponding status code] + } +} +``` + +5. **Time and memory limits.** +How are limits configured — per submission at request time, or set platform-wide? How does Judge0 surface timeouts and memory errors back through the API? + +6. **Python and Java support.** +Confirm both languages work end-to-end. Run submissions/add-two-numbers/correct.py and submissions/add-two-numbers-java/correct.java through your instance and verify you get the expected output. Note any differences in how the two languages behave. + +### 🖥 How to demonstrate it +Get a Judge0 instance running locally (Docker is fine) or use the public API for the demo. + +The demo should: + +1. Submit `submissions/add-two-numbers/correct.py` with the test file `questions/add-two-numbers/tests.py` to Judge0. +2. Poll for or receive the result. +3. Print the JSON output and a plain-text pass/fail verdict. +4. Repeat with `wrong-answer.py` and `broken.py` to confirm the failure and error cases surface correctly. +5. Repeat with the Java question (`submissions/add-two-numbers-java/`) to confirm Java works end-to-end. + +A simple script (Python, shell, or Node.js) hitting the Judge0 API is fine. No UI needed. + +Include a short written document covering your answers to the questions above, the actual latency numbers you measured, and a frank assessment of what it would take to keep this running as a volunteer-run service. + +### ✅ Is it usable for our case? +Determine yes, no, or yes with caveats. If caveats, list the important ones. This is the main thing the team needs from this ticket. ## Repo contents @@ -31,16 +79,4 @@ The goal is to have either a way to mark both Python and Java questions fully lo |---|---| | [SCHEMA.md](SCHEMA.md) | Question format specification | | [questions/](questions/) | Example questions in the defined schema | -| [submissions/](submissions/) | Sample student submissions used to test grading logic | - -## What is deliberately NOT here yet - -The following are out of scope until spike options have been evaluated and a direction is chosen: - -- A code editor (Monaco, CodeMirror, or otherwise) -- A UI shell or any frontend scaffolding -- A shared runner interface or abstraction layer -- Integration with the main Astro site -- Styling of any kind - -Spike authors should ideally not build any of the above. The purpose of each spike is narrow: determine whether the execution approach works and what its constraints are. UI and integration work follows after a direction is picked. +| [submissions/](submissions/) | Sample student submissions used to test grading logic | \ No newline at end of file diff --git a/Statuses.md b/Statuses.md new file mode 100644 index 0000000..8117fef --- /dev/null +++ b/Statuses.md @@ -0,0 +1,18 @@ +# Judge0 + +## Statuses + +01. In Queue +02. Processing +03. Accepted +04. Wrong Answer +05. Time Limit Exceeded +06. Compilation Error +07. Runtime Error (SIGSEGV) +08. Runtime Error (SIGXFSZ) +09. Runtime Error (SIGFPE) +10. Runtime Error (SIGABRT) +11. Runtime Error (NZEC) +12. Runtime Error (Other) +13. Internal Error +14. Exec Format Error diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..3480e01 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,47 @@ +x-logging: + &default-logging + logging: + driver: json-file + options: + max-size: 100M + +services: + server: + image: judge0/judge0:1.13.1 + volumes: + - ./judge0.conf:/judge0.conf:ro + ports: + - "2358:2358" + privileged: true + <<: *default-logging + restart: always + + workers: + image: judge0/judge0:1.13.1 + command: ["./scripts/workers"] + volumes: + - ./judge0.conf:/judge0.conf:ro + privileged: true + <<: *default-logging + restart: always + + db: + image: postgres:16.2 + env_file: judge0.conf + volumes: + - data:/var/lib/postgresql/data/ + <<: *default-logging + restart: always + + redis: + image: redis:7.2.4 + command: [ + "bash", "-c", + 'docker-entrypoint.sh --appendonly no --requirepass "$$REDIS_PASSWORD"' + ] + env_file: judge0.conf + <<: *default-logging + restart: always + +volumes: + data: diff --git a/judge0.conf b/judge0.conf new file mode 100644 index 0000000..c4afbf9 --- /dev/null +++ b/judge0.conf @@ -0,0 +1,358 @@ +################################################################################ +# Judge0 Configuration File +################################################################################ +# Judge0 is a highly configurable which allows you to use it for many +# different use-cases. Please, before deploying Judge0 take a look at this +# configuration file that is divided in several logical sections that will help +# you understand what can you do with Judge0. +# +# This default configuration file is designed to work out of the box for you and +# you can start with it when deploying Judge0 on your server. +# +# If you have any questions please don't hasitate to send an email or open an +# issue on the GitHub page of the project. + + +################################################################################ +# Judge0 Server Configuration +################################################################################ +# Enable or disable Judge0 Telemetry. +# Read more about it here: https://github.com/judge0/judge0/blob/master/TELEMETRY.md +# Default: true +JUDGE0_TELEMETRY_ENABLE= + +# Automatically restart Judge0 server if it fails to start. +# Default: 10 +RESTART_MAX_TRIES= + +# Maintenance mode is a mode in which clients cannot +# create or delete submissions while maintenance is enabled. +# Default: false +MAINTENANCE_MODE= + +# Set custom maintenance message that will be returned to clients +# who try to create or delete submisions. +# Default: Judge0 is currently in maintenance. +MAINTENANCE_MESSAGE= + +# If enabled user can request to synchronically wait for +# submission result on submission create. +# Default: true, i.e. user can request to wait for the result +ENABLE_WAIT_RESULT= + +# If enabled user is allowed to set custom compiler options. +# Default: true +ENABLE_COMPILER_OPTIONS= + +# List language names, separated by space, for which setting compiler options is allowed. +# Note that ENABLE_COMPILER_OPTIONS has higher priority, so this option will be +# ignored if setting compiler options is disabled with ENABLE_COMPILER_OPTIONS. +# For example, ALLOWED_LANGUAGES_FOR_COMPILER_OPTIONS="C C++ Java" would only +# allow setting compiler options for languages C, C++ and Java. +# Default: empty - for every compiled language setting compiler options is allowed. +ALLOWED_LANGUAGES_FOR_COMPILER_OPTIONS="Python Java" + +# If enabled user is allowed to set custom command line arguments. +# Default: true +ENABLE_COMMAND_LINE_ARGUMENTS= + +# If enabled autorized users can delete a submission. +# Default: false +ENABLE_SUBMISSION_DELETE= + +# If enabled user can GET and POST batched submissions. +# Default: true +ENABLE_BATCHED_SUBMISSIONS= + +# Maximum number of submissions that can be created or get in a batch. +# Default: 20 +MAX_SUBMISSION_BATCH_SIZE= + +# If enabled user can use callbacks. +# Default: true +ENABLE_CALLBACKS= + +# Maximum number of callback tries before giving up. +# Default: 3 +CALLBACKS_MAX_TRIES= + +# Timeout callback call after this many seconds. +# Default: 5 +CALLBACKS_TIMEOUT= + +# If enabled user can preset additional files in the sandbox. +# Default: true +ENABLE_ADDITIONAL_FILES= + +# Duration (in seconds) of submission cache. Decimal numbers are allowed. +# Set to 0 to turn of submission caching. Note that this does not apply to +# batched submissions. +# Default: 1 +SUBMISSION_CACHE_DURATION= + +# If true the documentation page will be used as a homepage, otherwise, the +# homepage will be empty. You can always access the documentation page via /docs. +# Default: false +USE_DOCS_AS_HOMEPAGE= + + +################################################################################ +# Judge0 Workers Configuration +################################################################################ +# Specify polling frequency in seconds. Decimal numbers are allowed. +# Default: 0.1 +INTERVAL= + +# Specify how many parallel workers to run. +# Default: 2*nproc (https://linux.die.net/man/1/nproc) +COUNT= + +# Specify maximum queue size. Represents maximum number of submissions that +# can wait in the queue at once. If request for new submission comes and the +# queue if full then submission will be rejected. +# Default: 100 +MAX_QUEUE_SIZE= + + +################################################################################ +# Judge0 Server Access Configuration +################################################################################ +# Allow only specified origins. +# If left blank, then all origins will be allowed (denoted with '*'). +# Example: +# ALLOW_ORIGIN="www.judge0.com judge0.com www.example.com blog.example.com" +ALLOW_ORIGIN= + +# Disallow only specified origins. +# If left blank, then no origin will be disallowed. +# Example: +# DISALLOW_ORIGIN="www.judge0.com judge0.com www.example.com blog.example.com" +DISALLOW_ORIGIN= + +# Allow only specified IP addresses. +# If left blank, then all IP addresses will be allowed. +# Example: +# ALLOW_IP="192.168.10.10 96.239.226.228 208.23.207.242" +ALLOW_IP= + +# Disallow only specified IP addresses. +# If left blank, then no IP addresses will be disallowed. +# Example: +# DISALLOW_IP="192.168.10.10 96.239.226.228 208.23.207.242" +DISALLOW_IP= + + +################################################################################ +# Judge0 Authentication Configuration +################################################################################ +# You can protect your API with (AUTHN_HEADER, AUTHN_TOKEN) pair. +# Each request then needs to have this pair either in headers or +# query parameters. For example let AUTHN_HEADER=X-Judge0-Token and +# AUTHN_TOKEN=mySecretToken. Then user should authenticate by sending this +# in headers or query parameters in each request, e.g.: +# https://api.judge0.com/system_info?X-Judge0-Token=mySecretToken + +# Specify authentication header name. +# Default: X-Auth-Token +AUTHN_HEADER= + +# Specify valid authentication tokens. +# Default: empty - authentication is disabled +AUTHN_TOKEN= + + +################################################################################ +# Judge0 Authorization Configuration +################################################################################ +# Protected API calls can be issued with (AUTHZ_HEADER, AUTHZ_TOKEN) pair. +# To see exactly which API calls are protected with authorization tokens +# please read the docs at https://api.judge0.com. +# API authorization ensures that only specified users call protected API calls. +# For example let AUTHZ_HEADER=X-Judge0-User and AUTHZ_TOKEN=mySecretToken. +# Then user should authorize be sending this in headers or query parameters in +# each request, e.g.: https://api.judge0.com/system_info?X-Judge0-User=mySecretToken +# Note that if you enabled authentication, then user should also send valid +# authentication token. + +# Specify authorization header name. +# Default: X-Auth-User +AUTHZ_HEADER= + +# Specify valid authorization tokens. +# Default: empty - authorization is disabled, protected API calls cannot be issued +AUTHZ_TOKEN= + + +################################################################################ +# Redis Configuration +################################################################################ +# Specify Redis host +# Default: localhost +REDIS_HOST=redis + +# Specify Redis port. +# Default: 6379 +REDIS_PORT= + +# Specify Redis password. Cannot be blank. +# Default: NO DEFAULT! MUST BE SET! +REDIS_PASSWORD=HjHX9t3Gkm6j7Y6dC2Hy5Jt8hkGdL3Ka + + +################################################################################ +# PostgreSQL Configuration +################################################################################ +# Specify Postgres host. +# Default: localhost +POSTGRES_HOST=db + +# Specify Postgres port. +# Default: 5432 +POSTGRES_PORT= + +# Name of the database to use. Used only in production. +# Default: postgres +POSTGRES_DB=judge0 + +# User who can access this database. Used only in production. +# Default: postgres +POSTGRES_USER=judge0 + +# Password of the user. Cannot be blank. Used only in production. +# Default: NO DEFAULT, YOU MUST SET YOUR PASSWORD +POSTGRES_PASSWORD=jEWWaCHvvFxSkWnGQPzn2rv5G27rZnTT + + +################################################################################ +# Submission Configuration +################################################################################ +# Judge0 uses isolate as an sandboxing environment. +# Almost all of the options you see here can be mapped to one of the options +# that isolate provides. For more information about these options please +# check for the isolate documentation here: +# https://raw.githubusercontent.com/ioi/isolate/master/isolate.1.txt + +# Default runtime limit for every program (in seconds). Decimal numbers are allowed. +# Time in which the OS assigns the processor to different tasks is not counted. +# Default: 5 +CPU_TIME_LIMIT= + +# Maximum custom CPU_TIME_LIMIT. +# Default: 15 +MAX_CPU_TIME_LIMIT= + +# When a time limit is exceeded, wait for extra time (in seconds), before +# killing the program. This has the advantage that the real execution time +# is reported, even though it slightly exceeds the limit. +# Default: 1 +CPU_EXTRA_TIME= + +# Maximum custom CPU_EXTRA_TIME. +# Default: 5 +MAX_CPU_EXTRA_TIME= + +# Limit wall-clock time in seconds. Decimal numbers are allowed. +# This clock measures the time from the start of the program to its exit, +# so it does not stop when the program has lost the CPU or when it is waiting +# for an external event. We recommend to use CPU_TIME_LIMIT as the main limit, +# but set WALL_TIME_LIMIT to a much higher value as a precaution against +# sleeping programs. +# Default: 10 +WALL_TIME_LIMIT= + +# Maximum custom WALL_TIME_LIMIT. +# Default: 20 +MAX_WALL_TIME_LIMIT= + +# Limit address space of the program in kilobytes. +# Default: 128000 +MEMORY_LIMIT= + +# Maximum custom MEMORY_LIMIT. +# Default: 512000 +MAX_MEMORY_LIMIT= + +# Limit process stack in kilobytes. +# Default: 64000 +STACK_LIMIT= + +# Maximum custom STACK_LIMIT. +# Default: 128000 +MAX_STACK_LIMIT= + +# Maximum number of processes and/or threads program can create. +# Default: 60 +MAX_PROCESSES_AND_OR_THREADS= + +# Maximum custom MAX_PROCESSES_AND_OR_THREADS. +# Default: 120 +MAX_MAX_PROCESSES_AND_OR_THREADS= + +# If true then CPU_TIME_LIMIT will be used as per process and thread. +# Default: false, i.e. CPU_TIME_LIMIT is set as a total limit for all processes and threads. +ENABLE_PER_PROCESS_AND_THREAD_TIME_LIMIT= + +# If false, user won't be able to set ENABLE_PER_PROCESS_AND_THREAD_TIME_LIMIT. +# Default: true +ALLOW_ENABLE_PER_PROCESS_AND_THREAD_TIME_LIMIT= + +# If true then MEMORY_LIMIT will be used as per process and thread. +# Default: false, i.e. MEMORY_LIMIT is set as a total limit for all processes and threads. +ENABLE_PER_PROCESS_AND_THREAD_MEMORY_LIMIT= + +# If false, user won't be able to set ENABLE_PER_PROCESS_AND_THREAD_MEMORY_LIMIT. +# Default: true +ALLOW_ENABLE_PER_PROCESS_AND_THREAD_MEMORY_LIMIT= + +# Limit size of files created (or modified) by the program in kilobytes. +# Default: 1024 +MAX_FILE_SIZE= + +# Maximum custom MAX_FILE_SIZE. +# Default: 4096 +MAX_MAX_FILE_SIZE= + +# Run each program this many times and take average of time and memory. +# Default: 1 +NUMBER_OF_RUNS= + +# Maximum custom NUMBER_OF_RUNS. +# Default: 20 +MAX_NUMBER_OF_RUNS= + +# Redirect stderr to stdout. +# Default: false +REDIRECT_STDERR_TO_STDOUT= + +# Maximum total size (in kilobytes) of extracted files from additional files archive. +# Default: 10240, i.e. maximum of 10MB in total can be extracted. +MAX_EXTRACT_SIZE= + +# If false, user won't be able to set ENABLE_NETWORK. +# Default: true, i.e. allow user to permit or deny network calls from the submission. +ALLOW_ENABLE_NETWORK= + +# If true submission will by default be able to do network calls. +# Default: false, i.e. programs cannot do network calls. +ENABLE_NETWORK= + + +################################################################################ +# Rails Configuration +################################################################################ +# Specify Rails environment: production or development +# Default: production +RAILS_ENV= + +# Specify maximum number of concurrent Rails threads. +# Default: nproc (https://linux.die.net/man/1/nproc) +RAILS_MAX_THREADS= + +# Specify how many processes will be created for handing requests. Each process +# will aditionally create RAILS_MAX_THREADS threads. +# Default: 2 +RAILS_SERVER_PROCESSES= + +# Secret key base for production, if not set it will be randomly generated +# Default: randomly generated +SECRET_KEY_BASE= diff --git a/questions/add-two-numbers/tests.py b/questions/add-two-numbers/tests.py index 120151c..fd8ee4e 100644 --- a/questions/add-two-numbers/tests.py +++ b/questions/add-two-numbers/tests.py @@ -1,5 +1,4 @@ import json -import solution results = {"passed": 0, "failed": 0, "failures": []} @@ -13,11 +12,11 @@ def run(name, got, expected): "message": f"expected {expected} got {got}", }) -run("test_positive", solution.add(1, 2), 3) -run("test_both_negative", solution.add(-3, -4), -7) -run("test_zero", solution.add(0, 0), 0) -run("test_mixed", solution.add(-1, 5), 4) -run("test_large", solution.add(100, 200), 300) +run("test_positive", add(1, 2), 3) +run("test_both_negative", add(-3, -4), -7) +run("test_zero", add(0, 0), 0) +run("test_mixed", add(-1, 5), 4) +run("test_large", add(100, 200), 300) print(json.dumps(results)) exit(results["failed"]) diff --git a/test_judge0.py b/test_judge0.py new file mode 100644 index 0000000..03ae4dd --- /dev/null +++ b/test_judge0.py @@ -0,0 +1,107 @@ +import base64, requests, json, time, sys + +def wait_for_submission(token, interval=0.5): + + poll_url = (f"http://localhost:2358/submissions/{token}" + "?base64_encoded=true" + ) + # url = f"http://localhost:2358/submissions/{token}" + + while True: + response = requests.get(poll_url) + response.raise_for_status() + + data = response.json() + status = data["status"]["id"] + + if status >= 3: + return data + + time.sleep(interval) + +def fetch_code(): + if len(sys.argv) < 2: + print("Usage: python submit.py [file2 ...]") + sys.exit(1) + + parts = [] + + # code-questions-test/submissions/add-two-numbers/correct.py + # code-questions-test/questions/add-two-numbers/tests.py + + try: + for filename in sys.argv[1:]: + with open(filename, "r", encoding='UTF-8') as f: + parts.append(f.read()) + + combined_code = "\n".join(parts) + + return combined_code + + except FileNotFoundError as e: + print(f"Could not find {e.filename}") + return "" + +source_code = fetch_code() +encoded_source = base64.b64encode(source_code.encode('utf-8')).decode('utf-8') +#print(encoded_source) + +# Define your target local URL and port (change 8080 to match your server's port) +url = "http://localhost:2358/submissions/?base64_encoded=true" + +# Define the data you want to send +payload = { + "source_code": encoded_source, + "language_id": "71", # Python3 = 71 + "number_of_runs": None, + "stdin": "Judge0", + "expected_output": None, + "cpu_time_limit": None, + "cpu_extra_time": None, + "wall_time_limit": None, + "memory_limit": None, + "stack_limit": None, + "max_processes_and_or_threads": None, + "enable_per_process_and_thread_time_limit": None, + "enable_per_process_and_thread_memory_limit": None, + "max_file_size": None, + "enable_network": None +} + +# Optional headers (useful for defining content types manually) +headers = { + "User-Agent": "PythonPostScript/1.0" +} + +try: + # 1. Sending data as JSON (Content-Type: application/json) + print("Sending JSON POST request...") + response = requests.post(url, json=payload, headers=headers) + + # Check if the request was successful + response.raise_for_status() + + # Print output responses + print(f"Status Code: {response.status_code}") + data = response.json() + token = data["token"] + print(f"Token: {token}") + + result = wait_for_submission(token) + + print(result["status"]["description"]) + if (result.get("stdout") != None): + print(base64.b64decode(result.get("stdout"))) + if (result.get("stderr") != None): + print(base64.b64decode(result.get("stderr"))) + #print(base64.b64decode(result.get("compile_output").encode()).decode()) + #print(result) + + +except requests.exceptions.ConnectionError: + print("Error: Could not connect to localhost. Is your local server running?") +except requests.exceptions.HTTPError as http_err: + print(f"HTTP error occurred: {http_err}") +except Exception as err: + print(f"An unexpected error occurred: {err}") + From eca6dcf33fc2b1b4ea89eabd8d6b83c12e20f35c Mon Sep 17 00:00:00 2001 From: DaveWJRH Date: Tue, 30 Jun 2026 02:11:29 -0400 Subject: [PATCH 2/2] Added additional documentation and findings --- .gitignore | 10 ++ Languages.md => LANGUAGES.md | 0 LINKS.md | 20 ++++ README.md | 166 ++++++++++++++++++++++++------ Statuses.md => STATUSES.md | 0 questions/reverse-string/tests.py | 11 +- test_judge0.py | 138 +++++++++++++------------ 7 files changed, 245 insertions(+), 100 deletions(-) rename Languages.md => LANGUAGES.md (100%) create mode 100644 LINKS.md rename Statuses.md => STATUSES.md (100%) diff --git a/.gitignore b/.gitignore index 48f3edd..3b666b7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,9 +10,19 @@ __pycache__/ *.pyo .venv/ venv/ +# +sort_json.py +tmp.py # Java *.class target/ build/ .gradle/ + +# JSON +languages.json + +# Folders +ide/ +judge0-v1.13.1.zip \ No newline at end of file diff --git a/Languages.md b/LANGUAGES.md similarity index 100% rename from Languages.md rename to LANGUAGES.md diff --git a/LINKS.md b/LINKS.md new file mode 100644 index 0000000..f718077 --- /dev/null +++ b/LINKS.md @@ -0,0 +1,20 @@ +# Judge0 + +## General +- [API](https://ce.judge0.com/) +- [Deployment](https://github.com/judge0/judge0/blob/master/CHANGELOG.md#deployment-procedure) + +## Localhost + +- [Dummy Client](http://localhost:2358/dummy-client) +- [API Docs](http://localhost:2358/docs) +- [Supported Languages](http://localhost:2358/languages) +- [Status Codes](http://localhost:2358/statuses) +- [System Information](http://localhost:2358/system_info) +- [Judge0 Configuration](http://localhost:2358/config_info) +- [Current Instance Stats](http://localhost:2358/statistics) +- [Workers Queue](http://localhost:2358/workers) +- [General Information](http://localhost:2358/about) +- [Judge0 Version](http://localhost:2358/version) +- [Judge0 Isolate Version](http://localhost:2358/isolate) +- [Judge0 License](http://localhost:2358/license) \ No newline at end of file diff --git a/README.md b/README.md index ae32b72..e44f7fa 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,134 @@ # Judge0 -API: **https://ce.judge0.com/** ## Goals -* Self-host a Judge0 instance. -* Submit a student's Python or Java submission alongside a test file. -* Get back pass/fail results matching the JSON format in `SCHEMA.md`. -* Understand what it takes to keep this running reliably. +✅ Self-host a Judge0 instance. +✅ Submit a student's Python or Java submission alongside a test file. +✅ Get back pass/fail results matching the JSON format in `SCHEMA.md`. +❔ Understand what it takes to keep this running reliably. ## Findings 1. **Setup complexity.** - What does self-hosting Judge0 actually involve? Walk through what you did: prerequisites, how long it took, and what was confusing or poorly documented. Also note whether the public hosted API is a realistic alternative — cover any rate limits, cost, or data concerns. +Here's some information on what is involved with self-hosting Judge0 based on my own setup experience: -2. **Infrastructure requirements.** -What kind of server does Judge0 need to run comfortably? What happens during a busy period — say, 100 students submitting simultaneously during a lab? What does it mean for a volunteer society to own and operate this long-term: keeping it running, handling updates, dealing with abuse, cost of hosting? +***Steps***: +Clone this repository to replicate the self-hosted judge0 instance I used, otherwise you can follow the instructions listed [here].(https://github.com/judge0/judge0/blob/master/CHANGELOG.md#deployment-procedure) +``` +git clone [URL] +cd [REPOSITORY] +``` +Run all services and wait a few seconds until everything is initialized: +``` +docker-compose up -d db redis +sleep 10s +docker-compose up -d +sleep 5s +``` +A Judge0 CE v1.13.1 instance is now up and running; visit docs at `http://:2358/docs` to confirm the instance is working as expected. + +To test python code execution, you can use the following commands: +``` +python3 test_judge0.py ./submissions/add-two-numbers/broken.py questions/add-two-numbers/tests.py + +python3 test_judge0.py ./submissions/add-two-numbers/correct.py questions/add-two-numbers/tests.py + +python3 test_judge0.py ./submissions/add-two-numbers/wrong-answer.py questions/add-two-numbers/tests.py + +python3 test_judge0.py ./submissions/reverse-string/broken.py questions/reverse-string/tests.py + +python3 test_judge0.py ./submissions/reverse-string/correct.py questions/reverse-string/tests.py + +python3 test_judge0.py ./submissions/reverse-string/wrong-answer.py questions/reverse-string/tests.py +``` +A sample of Java code submitted through the [Dummy Client](http://localhost:2358/dummy-client): +``` +import java.util.ArrayList; +import java.util.List; + +public class Main { + public static void main(String[] args) { + Tests.main(args); + } +} + +// SOLUTION (user provided) + +class Solution { + public static int add(int a, int b) { + return a + b; + } +} + +class Tests { + static int passed = 0; + static int failed = 0; + static List failures = new ArrayList<>(); + + static void run(String name, int got, int expected) { + if (got == expected) { + passed++; + } else { + failed++; + failures.add( + "{\"test\": \"" + name + "\", \"message\": \"expected " + expected + " got " + got + "\"}" + ); + } + } -***Docker Statistics*** + public static void main(String[] args) { + run("test_positive", Solution.add(1, 2), 3); + run("test_both_negative", Solution.add(-3, -4), -7); + run("test_zero", Solution.add(0, 0), 0); + run("test_mixed", Solution.add(-1, 5), 4); + run("test_large", Solution.add(100, 200), 300); + + StringBuilder sb = new StringBuilder(); + sb.append("{\"passed\": ").append(passed); + sb.append(", \"failed\": ").append(failed); + sb.append(", \"failures\": ["); + for (int i = 0; i < failures.size(); i++) { + if (i > 0) sb.append(", "); + sb.append(failures.get(i)); + } + sb.append("]}"); + System.out.println(sb.toString()); + System.exit(failed); + } +} +``` + +***Prerequisites***: +- Judge0 Docker image size: 3.295GB +- Base storage: 20GB +- Recommended storage: >=50GB +- Base RAM: 2GB +- Recommended RAM: >=4GB +- Base CPU: 2vCPU +- Recommended CPU: >=4vCPU + +***Time***: +- Editing judge0 (CONF file) & docker compose (YAML source): ≈15 mins +- Judge0 works well with minimum technical setup requirements, making deployment fairly quick. + +***Potential Roadblocks***: +- I appended two python scripts together so they are submitted as a single file for code execution. This means that I had to tweak the test file so that it wasn't importing the user submission, it simply used the user-defined functions within the test suite as a single file. +- Judge0 has only been tested on Linux and might not work well on other systems (if at all) +- Implementing some kind of UI, or using an external code editor (Monaco, CodeMirror, Ace), will require development as well as regular maintenance. + +***Hosted API***: +- RapidAPI + - Basic: $0.0017/use + - Pro: $44.99/mo (2000 submissions/day) + - Ultra: $89.99/mo (5000 submissions/day) + - Mega: $169.99/mo (10000 submissions/day) + - \+ Bandwidth Platform fee (10240MB/mo + $0.001/ per 1MB) + +2. **Infrastructure requirements.** +Further testing is needed to determine any issues that might arise during spikes in traffic. +Implementing a pseudo-IDE or code editor may take some time for developers, based on server/hardware restrictions, instance requirements, and potentially refactoring existing website code. Security/software updates will be dependent on the latest Docker image. And an effective method to prevent misuse (ie: input validation, infinite loops, bots) will need to be considered - on top of what can be mitigated through the Judge0 configuration file. + + +***My Docker Statistics*** - Container CPU Usage: 12%/1200% (12 CPUs available) With 20 runs, 200%/1200% - Container memory usage: 2.28GB/15.17GB @@ -28,7 +141,8 @@ With 20 runs, 200%/1200% - redis (redis:7.2.4) 3. **End-to-end latency.** -From submission sent to result received, how long does it take? Break it down: queue wait, execution time, API round-trip. What's the experience like for a student waiting on feedback? +Average Code Execution Time: 0.02 seconds +More testing needs to be done for concurrent submissions, queuing, and multi-user wait times. 4. **Question and submission workflow.** A test file and the corresponding submission code can be appended together to execute the test suite using the user provided solution. The 200 OK Response will be JSON formatted to include: @@ -48,35 +162,27 @@ A test file and the corresponding submission code can be appended together to ex } } ``` +5. **Time and memory limits.** +The `judge0.conf` file allows you to customize server, workers, server access, authorization, redis, postgreSQL, submission, and rails configurations. For submissions, platform-wide settings include CPU time, max memory, max stack, max processes/threads, file size, number of runs, and more. It is also possible to tweak these settings per submission using JSON formatted HTTP requests - these can be added to specific submissions based on how it is implemented within the question repository. +Judge0 surfaces timeouts and memory errors back through the API within the 200 OK Response using JSON formatting to list `stdout`, `stderr`, `message`, and `status` fields. -5. **Time and memory limits.** -How are limits configured — per submission at request time, or set platform-wide? How does Judge0 surface timeouts and memory errors back through the API? - -6. **Python and Java support.** -Confirm both languages work end-to-end. Run submissions/add-two-numbers/correct.py and submissions/add-two-numbers-java/correct.java through your instance and verify you get the expected output. Note any differences in how the two languages behave. - -### 🖥 How to demonstrate it -Get a Judge0 instance running locally (Docker is fine) or use the public API for the demo. - -The demo should: - -1. Submit `submissions/add-two-numbers/correct.py` with the test file `questions/add-two-numbers/tests.py` to Judge0. -2. Poll for or receive the result. -3. Print the JSON output and a plain-text pass/fail verdict. -4. Repeat with `wrong-answer.py` and `broken.py` to confirm the failure and error cases surface correctly. -5. Repeat with the Java question (`submissions/add-two-numbers-java/`) to confirm Java works end-to-end. +6. **Python and Java support.** +A python script has been created for the purposes of testing Python code execution, while I have manually used the [Localhost Dummy Client](http://localhost:2358/dummy-client) to verify Java code execution (similar method of combining the user submission and the test files). Compiler flags and command line arguments can also be added to submissions within the POST request. -A simple script (Python, shell, or Node.js) hitting the Judge0 API is fine. No UI needed. +\* Make sure to update the variables `REDIS_PASSWORD` and `POSTGRES_PASSWORD` in the `judge0.conf` file before deploying to a production environment. -Include a short written document covering your answers to the questions above, the actual latency numbers you measured, and a frank assessment of what it would take to keep this running as a volunteer-run service. ### ✅ Is it usable for our case? -Determine yes, no, or yes with caveats. If caveats, list the important ones. This is the main thing the team needs from this ticket. +Depending on available computing resources for CCSS, I believe that this option is feasible (with the caveats listed above), but would need further testing to determine weakpoints during periods of high traffic (ie: during exams). ## Repo contents | Path | Description | |---|---| | [SCHEMA.md](SCHEMA.md) | Question format specification | +| [LANGUAGES.md](LANGUAGES.md) | Supported/Active Languages | +| [LINKS.md](LINKS.md) | Documentation & Localhost Links | +| [STATUSES.md](STATUSES.md) | Status codes returned after code execution | | [questions/](questions/) | Example questions in the defined schema | -| [submissions/](submissions/) | Sample student submissions used to test grading logic | \ No newline at end of file +| [submissions/](submissions/) | Sample student submissions used to test grading logic | +| [Python Test Script](test_judge0.py) | Reads an example python code submission as well as the corresponding tester file, appending them together and submitting as a JSON payload to the Judge0 instance for code execution | \ No newline at end of file diff --git a/Statuses.md b/STATUSES.md similarity index 100% rename from Statuses.md rename to STATUSES.md diff --git a/questions/reverse-string/tests.py b/questions/reverse-string/tests.py index 33b32c9..f6c740c 100644 --- a/questions/reverse-string/tests.py +++ b/questions/reverse-string/tests.py @@ -1,5 +1,4 @@ import json -import solution results = {"passed": 0, "failed": 0, "failures": []} @@ -13,11 +12,11 @@ def run(name, got, expected): "message": f"expected {repr(expected)} got {repr(got)}", }) -run("test_empty", solution.reverse(""), "") -run("test_single", solution.reverse("a"), "a") -run("test_normal", solution.reverse("hello"), "olleh") -run("test_palindrome", solution.reverse("racecar"), "racecar") -run("test_unicode", solution.reverse("café"), "éfac") +run("test_empty", reverse(""), "") +run("test_single", reverse("a"), "a") +run("test_normal", reverse("hello"), "olleh") +run("test_palindrome", reverse("racecar"), "racecar") +run("test_unicode", reverse("café"), "éfac") print(json.dumps(results)) exit(results["failed"]) diff --git a/test_judge0.py b/test_judge0.py index 03ae4dd..60bd404 100644 --- a/test_judge0.py +++ b/test_judge0.py @@ -21,11 +21,12 @@ def wait_for_submission(token, interval=0.5): def fetch_code(): if len(sys.argv) < 2: - print("Usage: python submit.py [file2 ...]") + print("Usage: python3 test_judge0.py ") sys.exit(1) parts = [] + # Example files to be appended: # code-questions-test/submissions/add-two-numbers/correct.py # code-questions-test/questions/add-two-numbers/tests.py @@ -42,66 +43,75 @@ def fetch_code(): print(f"Could not find {e.filename}") return "" -source_code = fetch_code() -encoded_source = base64.b64encode(source_code.encode('utf-8')).decode('utf-8') -#print(encoded_source) - -# Define your target local URL and port (change 8080 to match your server's port) -url = "http://localhost:2358/submissions/?base64_encoded=true" - -# Define the data you want to send -payload = { - "source_code": encoded_source, - "language_id": "71", # Python3 = 71 - "number_of_runs": None, - "stdin": "Judge0", - "expected_output": None, - "cpu_time_limit": None, - "cpu_extra_time": None, - "wall_time_limit": None, - "memory_limit": None, - "stack_limit": None, - "max_processes_and_or_threads": None, - "enable_per_process_and_thread_time_limit": None, - "enable_per_process_and_thread_memory_limit": None, - "max_file_size": None, - "enable_network": None -} - -# Optional headers (useful for defining content types manually) -headers = { - "User-Agent": "PythonPostScript/1.0" -} - -try: - # 1. Sending data as JSON (Content-Type: application/json) - print("Sending JSON POST request...") - response = requests.post(url, json=payload, headers=headers) - - # Check if the request was successful - response.raise_for_status() - - # Print output responses - print(f"Status Code: {response.status_code}") - data = response.json() - token = data["token"] - print(f"Token: {token}") - - result = wait_for_submission(token) - - print(result["status"]["description"]) - if (result.get("stdout") != None): - print(base64.b64decode(result.get("stdout"))) - if (result.get("stderr") != None): - print(base64.b64decode(result.get("stderr"))) - #print(base64.b64decode(result.get("compile_output").encode()).decode()) - #print(result) - - -except requests.exceptions.ConnectionError: - print("Error: Could not connect to localhost. Is your local server running?") -except requests.exceptions.HTTPError as http_err: - print(f"HTTP error occurred: {http_err}") -except Exception as err: - print(f"An unexpected error occurred: {err}") - +def main(): + # Read user submission & test suite + source_code = fetch_code() + + encoded_source = base64.b64encode(source_code.encode('utf-8')).decode('utf-8') + + # Define your target local URL and port (change 8080 to match your server's port) + url = "http://localhost:2358/submissions/?base64_encoded=true" + + # Define the data you want to send + payload = { + "source_code": encoded_source, + "language_id": "71", # Python3 = 71, Java = 62 + "number_of_runs": None, + "stdin": "Judge0", + "expected_output": None, + "cpu_time_limit": None, + "cpu_extra_time": None, + "wall_time_limit": None, + "memory_limit": None, + "stack_limit": None, + "max_processes_and_or_threads": None, + "enable_per_process_and_thread_time_limit": None, + "enable_per_process_and_thread_memory_limit": None, + "max_file_size": None, + "enable_network": None + } + + # Optional headers (useful for defining content types manually) + headers = { + "User-Agent": "PythonPostScript/1.0" + } + + count = 0 + tests_to_queue = 10 + + while(count < tests_to_queue): + try: + count += 1 + # 1. Sending data as JSON (Content-Type: application/json) + print("Sending JSON POST request...") + response = requests.post(url, json=payload, headers=headers) + + # Check if the request was successful + response.raise_for_status() + + # Print output responses + print(f"Status Code: {response.status_code}") + data = response.json() + token = data["token"] + print(f"Token: {token}") + + result = wait_for_submission(token) + + print(result["status"]["description"]) + if (result.get("stdout") != None): + print(base64.b64decode(result.get("stdout"))) + if (result.get("stderr") != None): + print(base64.b64decode(result.get("stderr"))) + #print(base64.b64decode(result.get("compile_output").encode()).decode()) + #print(result) + + except requests.exceptions.ConnectionError: + print("Error: Could not connect to localhost. Is your local server running?") + except requests.exceptions.HTTPError as http_err: + print(f"HTTP error occurred: {http_err}") + except Exception as err: + print(f"An unexpected error occurred: {err}") + + +if __name__ == "__main__": + main() \ No newline at end of file