diff options
25 files changed, 341 insertions, 31 deletions
@@ -1,7 +1,8 @@ # We run various services to simulate a production environment. +# Note that MINIO is not ignored, as we need the data for complete setup (just liket he DB dump). local/beanstalkd/ -local/minio/ local/postgresql/ +local/minio/.minio.sys # Ignore generated files for shell environment. .direnv/ diff --git a/app/package-lock.json b/app/package-lock.json index 1614662..a06705e 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "dependencies": { "@aws-sdk/client-s3": "^3.750.0", + "beanstalkd": "^2.2.5", "pg": "^8.13.3" }, "devDependencies": { @@ -17,6 +18,7 @@ "@sveltejs/adapter-auto": "^4.0.0", "@sveltejs/kit": "^2.16.0", "@sveltejs/vite-plugin-svelte": "^5.0.0", + "@types/beanstalkd": "^2.2.5", "@types/pg": "^8.11.11", "eslint": "^9.18.0", "eslint-config-prettier": "^10.0.1", @@ -2723,6 +2725,16 @@ "vite": "^6.0.0" } }, + "node_modules/@types/beanstalkd": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/@types/beanstalkd/-/beanstalkd-2.2.5.tgz", + "integrity": "sha512-DxZdGjeEgQNohuBNUL42dKkQsImXZVsBWDO5Ylvk98CZrQ9MF7cBvUWfxKf0kRZ/SgsY4ip1vZV0vjAhg17Y1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/cookie": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", @@ -3127,6 +3139,15 @@ "node": ">= 0.4" } }, + "node_modules/babel-runtime": { + "version": "5.8.38", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-5.8.38.tgz", + "integrity": "sha512-KpgoA8VE/pMmNCrnEeeXqFG24TIH11Z3ZaimIhJWsin8EbfZy3WzFKUTIan10ZIDgRVvi9EkLbruJElJC9dRlg==", + "license": "MIT", + "dependencies": { + "core-js": "^1.0.0" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -3134,6 +3155,69 @@ "dev": true, "license": "MIT" }, + "node_modules/beanstalkd": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/beanstalkd/-/beanstalkd-2.2.5.tgz", + "integrity": "sha512-OqptEiFBQl8lYQKuJGTwwyKKcpWbP0vtbZWAPLWcymxAgqNgst7Y6R+2GHfTknzOtJ5b8Su8Yx1f3MAsf4z92A==", + "license": "MIT", + "dependencies": { + "babel-runtime": "^5.8.25", + "beanstalkd-protocol": "^1.0.1", + "bluebird": "^3.4.7", + "debug": "^2.2.0", + "js-yaml": "^3.13.1", + "lodash.camelcase": "^4.3.0" + } + }, + "node_modules/beanstalkd-protocol": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/beanstalkd-protocol/-/beanstalkd-protocol-1.0.1.tgz", + "integrity": "sha512-3LVZru/dWiYMjb+CNmYOuf2Xy1rgy4WsomEzIoZVclCXxzwwncxTl9TUUyJOO8IJ2GB5r7BWohBKBYxMAU9ysA==", + "license": "MIT" + }, + "node_modules/beanstalkd/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/beanstalkd/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/beanstalkd/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/beanstalkd/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "license": "MIT" + }, "node_modules/bowser": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", @@ -3254,6 +3338,13 @@ "node": ">= 0.6" } }, + "node_modules/core-js": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz", + "integrity": "sha512-ZiPp9pZlgxpWRu0M+YWbm6+aQ84XEfH1JRXvfOc/fILWI0VKhLC2LX13X1NYq4fULzLMq7Hfh43CSo2/aIaUPA==", + "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.", + "license": "MIT" + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -3557,6 +3648,19 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esquery": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", @@ -4039,6 +4143,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -4799,6 +4909,12 @@ "node": ">= 10.x" } }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", diff --git a/app/package.json b/app/package.json index 1e5986e..873d5c8 100644 --- a/app/package.json +++ b/app/package.json @@ -19,6 +19,7 @@ "@sveltejs/adapter-auto": "^4.0.0", "@sveltejs/kit": "^2.16.0", "@sveltejs/vite-plugin-svelte": "^5.0.0", + "@types/beanstalkd": "^2.2.5", "@types/pg": "^8.11.11", "eslint": "^9.18.0", "eslint-config-prettier": "^10.0.1", @@ -34,6 +35,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.750.0", + "beanstalkd": "^2.2.5", "pg": "^8.13.3" } } diff --git a/app/src/app.d.ts b/app/src/app.d.ts index 6dba3f1..71e7b15 100644 --- a/app/src/app.d.ts +++ b/app/src/app.d.ts @@ -1,5 +1,6 @@ import type { User } from "$lib/server/users"; import type { S3Client } from "@aws-sdk/client-s3"; +import type BeanstalkdClient from "beanstalkd"; import type { PoolClient } from "pg"; // See https://svelte.dev/docs/kit/types#app.d.ts @@ -12,6 +13,8 @@ declare global { s3Client: S3Client; + beanstalkdClient: BeanstalkdClient; + /** * The user, if they are logged in. * diff --git a/app/src/hooks.server.ts b/app/src/hooks.server.ts index f342e03..78342cd 100644 --- a/app/src/hooks.server.ts +++ b/app/src/hooks.server.ts @@ -1,21 +1,23 @@ -import { getDbConnection } from "$lib/server/db"; +import { getDbClient } from "$lib/server/db"; import { getS3Client } from "$lib/server/s3"; import { validateSessionToken } from "$lib/server/sessions"; import { type Handle } from "@sveltejs/kit"; import { sequence } from "@sveltejs/kit/hooks"; +import { beanstalkdHandle } from "$lib/server/beanstalkd"; const dbHandle = (async ({ event, resolve }) => { - const dbConn = await getDbConnection(); - event.locals.dbConn = dbConn; + const dbClient = await getDbClient(); + event.locals.dbClient = dbClient; try { return await resolve(event); } finally { - dbConn.release(); + dbClient.release(); } }) satisfies Handle; // FIXME: Kind of stupid to load for every request. Should probs move handler to $lib and import for relevant routes. +// Same goes for beanstalkd. const s3Handle = (async ({ event, resolve }) => { const s3Client = getS3Client(); event.locals.s3Client = s3Client; @@ -39,4 +41,5 @@ const sessionHandle = (async ({ event, resolve }) => { return resolve(event); }) satisfies Handle; -export const handle = sequence(dbHandle, s3Handle, sessionHandle); +// FIXME: Kind of stupid to load for every request. Should probs move handler to $lib and import for relevant routes. +export const handle = sequence(dbHandle, s3Handle, beanstalkdHandle, sessionHandle); diff --git a/app/src/lib/common/assignments.ts b/app/src/lib/common/assignments.ts index 79553cd..6e9d574 100644 --- a/app/src/lib/common/assignments.ts +++ b/app/src/lib/common/assignments.ts @@ -7,12 +7,12 @@ export interface CemetaryPlot { } /** Corresponds to `assignment_state`. */ -export type AssignmentState = "AWAITING_GARDENER_NOTIFICATION" - | "AWAITING_FINISH" - | "AWAITING_WATERMARKING" - | "AWAITING_OWNER_NOTIFICATION" - | "DONE" - ; +export type AssignmentState = + | "AWAITING_GARDENER_NOTIFICATION" + | "AWAITING_FINISH" + | "AWAITING_WATERMARKING" + | "AWAITING_OWNER_NOTIFICATION" + | "DONE"; /** A row from the `assignments` table. */ export interface Assignment { @@ -23,10 +23,10 @@ export interface Assignment { state: AssignmentState; } - // FIXME: We have ORM at home. A more clearly defined (OOP) model layer. /** Checks whether the state of the assignment allows it to be "finished". */ export function canBeFinished(assignment: Assignment): boolean { - return (assignment.state === "AWAITING_GARDENER_NOTIFICATION" || - assignment.state === "AWAITING_FINISH") + return ( + assignment.state === "AWAITING_GARDENER_NOTIFICATION" || assignment.state === "AWAITING_FINISH" + ); } diff --git a/app/src/lib/server/assignments.ts b/app/src/lib/server/assignments.ts index c66f25c..9c28cff 100644 --- a/app/src/lib/server/assignments.ts +++ b/app/src/lib/server/assignments.ts @@ -1,6 +1,7 @@ -import { PutObjectCommand, type S3Client } from "@aws-sdk/client-s3"; /*=;* / +import { PutObjectCommand, type S3Client } from "@aws-sdk/client-s3"; /*=;*/ import type { ClientBase } from "pg"; -import type { CemetaryPlot, Assignment, AssignmentState } from "../common/assignments"; +import type { CemetaryPlot, Assignment } from "../common/assignments"; +import type BeanstalkdClient from "beanstalkd"; /** * Retrieves all assignments for the given user. @@ -73,17 +74,24 @@ export async function getAssignmentAndCemetaryById( } export interface FinishAssignmentArgs { + dbClient: ClientBase; + s3Client: S3Client; + beanstalkdClient: BeanstalkdClient; + images: { bytes: Uint8Array; name: string }[]; note?: string; assignmentId: number; } // TODO: Error recovery. -export async function finishAssignment( - dbClient: ClientBase, - s3Client: S3Client, - { images, note, assignmentId }: FinishAssignmentArgs, -): Promise<void> { +export async function finishAssignment({ + dbClient, + s3Client, + beanstalkdClient, + images, + note, + assignmentId, +}: FinishAssignmentArgs): Promise<void> { // Upload to S3, returning path // FIXME: Should be factored out? const uploadPromises = images.map(async (image) => { @@ -101,7 +109,10 @@ export async function finishAssignment( }); const uploadedImages = await Promise.all(uploadPromises); - // TODO: Add beanstalkd job + // Instruct background worker to do watermarking. + // FIXME: magic constants yay how fun + await beanstalkdClient.use("watermarking"); + console.debug(await beanstalkdClient.put(0, 0, 60, JSON.stringify({ assignmentId }))); await dbClient.query("BEGIN"); diff --git a/app/src/lib/server/beanstalkd.ts b/app/src/lib/server/beanstalkd.ts new file mode 100644 index 0000000..10f392a --- /dev/null +++ b/app/src/lib/server/beanstalkd.ts @@ -0,0 +1,19 @@ +import pkg from "beanstalkd"; +import type { Handle } from "@sveltejs/kit"; + +// Annoying CommonJS interop issue (vitejs/vite#2139) which combines with incorrect typings from DefinitelyTyped project :( +// @ts-ignore +const BeanstalkdClient = pkg.default as typeof pkg; + +export const beanstalkdHandle = (async ({ event, resolve }) => { + // FIXME: Should obv. read from env. + const beanstalkdClient = new BeanstalkdClient("localhost", 11300); + await beanstalkdClient.connect(); + + event.locals.beanstalkdClient = beanstalkdClient; + try { + return await resolve(event); + } finally { + beanstalkdClient.quit(); + } +}) satisfies Handle; diff --git a/app/src/routes/assignments/+page.svelte b/app/src/routes/assignments/+page.svelte index f43767d..5a3a4cb 100644 --- a/app/src/routes/assignments/+page.svelte +++ b/app/src/routes/assignments/+page.svelte @@ -1,15 +1,19 @@ <!-- The /assignments index page gives an overview of assignments --> <script lang="ts"> + import { canBeFinished } from "$lib/common/assignments"; import type { PageProps } from "./$types"; const { data }: PageProps = $props(); + const unfinishedAssignments = $derived(data.assignments.filter(canBeFinished)); + const finishedAssignments = $derived(data.assignments.filter((a) => !canBeFinished(a))); </script> -<h1>Kommende opgaver for {data.user.firstName}</h1> +<h1>{data.user.firstName}s opgaver</h1> +<h2>Kommende opgaver</h2> <ol> - {#each data.assignments as assignment} + {#each unfinishedAssignments as assignment} <li> <span>{assignment.date}</span> <a href={`/assignments/${assignment.id}`}>Mere info</a> @@ -17,5 +21,12 @@ {/each} </ol> -<style> -</style> +<h2>Færdiggjorte opgaver</h2> +<ol> + {#each finishedAssignments as assignment} + <li> + <span>{assignment.date}</span> + <a href={`/assignments/${assignment.id}`}>Mere info</a> + </li> + {/each} +</ol> diff --git a/app/src/routes/assignments/[assignmentId]/+page.server.ts b/app/src/routes/assignments/[assignmentId]/+page.server.ts index 3b53a7c..b6634f0 100644 --- a/app/src/routes/assignments/[assignmentId]/+page.server.ts +++ b/app/src/routes/assignments/[assignmentId]/+page.server.ts @@ -42,7 +42,13 @@ export const actions = { })), ); - await finishAssignment(locals.dbClient, locals.s3Client, { + const { beanstalkdClient, dbClient, s3Client } = locals; + + await finishAssignment({ + beanstalkdClient, + dbClient, + s3Client, + images, note, assignmentId: +params.assignmentId, // We have parsing at home... diff --git a/app/src/routes/assignments/[assignmentId]/+page.svelte b/app/src/routes/assignments/[assignmentId]/+page.svelte index 6b24f38..bf955f9 100644 --- a/app/src/routes/assignments/[assignmentId]/+page.svelte +++ b/app/src/routes/assignments/[assignmentId]/+page.svelte @@ -30,7 +30,9 @@ </label> <label> Ekstra bemærkninger: - <textarea name="notes" placeholder="F.eks.: Vi løb tør for roser (?). De kommer i overmorgen :)" + <textarea + name="notes" + placeholder="F.eks.: Vi løb tør for roser (?). De kommer i overmorgen :)" ></textarea> </label> <button>Færddigør job</button> diff --git a/db/run_db.sh b/db/run_db.sh index 3fdf58f..026fb1e 100755 --- a/db/run_db.sh +++ b/db/run_db.sh @@ -1,2 +1,2 @@ set -ue -postgres -D "$1" -c listen_addresses=localhost +postgres -D "$1" -c listen_addresses=localhost -c log_statement=all @@ -16,7 +16,9 @@ devShells = eachNixpkgs (pkgs: { default = pkgs.mkShell { inputsFrom = [ - # self.packages.${pkgs.system}.default + # self.packages.${pkgs.system}.app + # self.packages.${pkgs.system}.watermark-worker + # self.packages.${pkgs.system}.notification-worker ]; buildInputs = with pkgs; [ # Services needed for a full deployment. @@ -27,6 +29,14 @@ # Development utilities. minio-client modd + + # Python interpreter + (python3.withPackages (ps: with ps; [ + pillow + psycopg + boto3 + beanstalkc + ])) ]; }; }); diff --git a/local/minio/images/1f10dc53-3652-45fd-9733-3956cef83c1e/b735c14b-0e78-4cb5-95cc-e10e9c4bf08f/part.1 b/local/minio/images/1f10dc53-3652-45fd-9733-3956cef83c1e/b735c14b-0e78-4cb5-95cc-e10e9c4bf08f/part.1 Binary files differnew file mode 100644 index 0000000..f089ad3 --- /dev/null +++ b/local/minio/images/1f10dc53-3652-45fd-9733-3956cef83c1e/b735c14b-0e78-4cb5-95cc-e10e9c4bf08f/part.1 diff --git a/local/minio/images/1f10dc53-3652-45fd-9733-3956cef83c1e/xl.meta b/local/minio/images/1f10dc53-3652-45fd-9733-3956cef83c1e/xl.meta Binary files differnew file mode 100644 index 0000000..cd73878 --- /dev/null +++ b/local/minio/images/1f10dc53-3652-45fd-9733-3956cef83c1e/xl.meta diff --git a/local/minio/images/6891d2b7-e70f-40b4-9b00-91f0a7328dcb/b5ed3cb7-8cea-41c6-9281-c4a323c10937/part.1 b/local/minio/images/6891d2b7-e70f-40b4-9b00-91f0a7328dcb/b5ed3cb7-8cea-41c6-9281-c4a323c10937/part.1 Binary files differnew file mode 100644 index 0000000..ebf4e18 --- /dev/null +++ b/local/minio/images/6891d2b7-e70f-40b4-9b00-91f0a7328dcb/b5ed3cb7-8cea-41c6-9281-c4a323c10937/part.1 diff --git a/local/minio/images/6891d2b7-e70f-40b4-9b00-91f0a7328dcb/xl.meta b/local/minio/images/6891d2b7-e70f-40b4-9b00-91f0a7328dcb/xl.meta Binary files differnew file mode 100644 index 0000000..c05e343 --- /dev/null +++ b/local/minio/images/6891d2b7-e70f-40b4-9b00-91f0a7328dcb/xl.meta diff --git a/local/minio/images/6e61c938-d6fe-4558-a4bd-c248708997b4/2651427f-23da-4aeb-8707-53eddb386303/part.1 b/local/minio/images/6e61c938-d6fe-4558-a4bd-c248708997b4/2651427f-23da-4aeb-8707-53eddb386303/part.1 Binary files differnew file mode 100644 index 0000000..24c143c --- /dev/null +++ b/local/minio/images/6e61c938-d6fe-4558-a4bd-c248708997b4/2651427f-23da-4aeb-8707-53eddb386303/part.1 diff --git a/local/minio/images/6e61c938-d6fe-4558-a4bd-c248708997b4/xl.meta b/local/minio/images/6e61c938-d6fe-4558-a4bd-c248708997b4/xl.meta Binary files differnew file mode 100644 index 0000000..7babedb --- /dev/null +++ b/local/minio/images/6e61c938-d6fe-4558-a4bd-c248708997b4/xl.meta diff --git a/local/minio/images/e843d334-39f6-4b95-864a-ab8d9ef25bb8/fabb2def-4f25-4414-a5bd-75b911b16d19/part.1 b/local/minio/images/e843d334-39f6-4b95-864a-ab8d9ef25bb8/fabb2def-4f25-4414-a5bd-75b911b16d19/part.1 Binary files differnew file mode 100644 index 0000000..68f1b3e --- /dev/null +++ b/local/minio/images/e843d334-39f6-4b95-864a-ab8d9ef25bb8/fabb2def-4f25-4414-a5bd-75b911b16d19/part.1 diff --git a/local/minio/images/e843d334-39f6-4b95-864a-ab8d9ef25bb8/xl.meta b/local/minio/images/e843d334-39f6-4b95-864a-ab8d9ef25bb8/xl.meta Binary files differnew file mode 100644 index 0000000..b9ed01f --- /dev/null +++ b/local/minio/images/e843d334-39f6-4b95-864a-ab8d9ef25bb8/xl.meta diff --git a/local/minio/images/fe5e7c27-99c4-4d4b-ba77-fcad5c6607e3/0e35331a-332f-4d3d-9019-55ecbd356578/part.1 b/local/minio/images/fe5e7c27-99c4-4d4b-ba77-fcad5c6607e3/0e35331a-332f-4d3d-9019-55ecbd356578/part.1 Binary files differnew file mode 100644 index 0000000..ebf4e18 --- /dev/null +++ b/local/minio/images/fe5e7c27-99c4-4d4b-ba77-fcad5c6607e3/0e35331a-332f-4d3d-9019-55ecbd356578/part.1 diff --git a/local/minio/images/fe5e7c27-99c4-4d4b-ba77-fcad5c6607e3/xl.meta b/local/minio/images/fe5e7c27-99c4-4d4b-ba77-fcad5c6607e3/xl.meta Binary files differnew file mode 100644 index 0000000..15b1243 --- /dev/null +++ b/local/minio/images/fe5e7c27-99c4-4d4b-ba77-fcad5c6607e3/xl.meta @@ -11,6 +11,10 @@ app/package.json app/*config* { daemon: npm run dev } +watermark_worker/*.py { + daemon: python3 watermark_worker/watermark.py +} + # Database server db/*.sh { prep: bash db/create_db.sh ./local/postgresql diff --git a/watermark_worker/watermark.py b/watermark_worker/watermark.py new file mode 100644 index 0000000..f2b02f0 --- /dev/null +++ b/watermark_worker/watermark.py @@ -0,0 +1,122 @@ +import PIL.Image +import PIL.ImageDraw +import PIL.ImageFont +import boto3 +from botocore.credentials import json +import psycopg +import io +import tempfile +import beanstalkc + +# FIXME: These connection functions should obviously load the correct data in a full system. + +def connect_to_db(): + return psycopg.connect("dbname=testdb user=postgres") + +def connect_to_s3(): + return boto3.client("s3", + endpoint_url="http://localhost:9000", + aws_access_key_id="minioadmin", + aws_secret_access_key="minioadmin", + verify=False) # disable SSL + +def connect_to_beanstalkd(): + return beanstalkc.Connection("localhost", 11300) + +def watermark(image: PIL.Image.Image) -> PIL.Image.Image: + w, h = image.size + padding = 0.05 * min(w, h) + x = padding + y = h - padding + font_size = 0.1 * min(w, h) + + watermark_image = image.copy() + draw = PIL.ImageDraw.ImageDraw(watermark_image) + draw.text((x, y), "Memmora", font_size=font_size, anchor="ls") + + return watermark_image + +def process_image(db_conn: psycopg.Connection, s3_client, image_id: int, s3_key: str): + IMAGES_BUCKET = "images" + + print(f"Processing image {image_id} ({s3_key})...") + + # Retrieve image from S3 + print(">> Retrieving image") + original_image_bytes = io.BytesIO() + s3_client.download_fileobj(IMAGES_BUCKET, s3_key, original_image_bytes) + + # Process image + print(">> Watermarking image") + original_image = PIL.Image.open(original_image_bytes) + watermarked_image = watermark(original_image) + watermarked_image_bytes = io.BytesIO() + watermarked_image.save(watermarked_image_bytes, format="png") + + # Update S3 object + # FIXME: API sucks ass, avoid writing tmp file. At least /tmp should be ramfs. + print(">> Uploading watermarked image to S3") + with tempfile.NamedTemporaryFile("wb", delete_on_close=False) as fp: + fp.write(watermarked_image_bytes.getbuffer()) + fp.close() + s3_client.upload_file(fp.name, IMAGES_BUCKET, s3_key) + + # Mark the image as processed in the Database + # FIXME: If this step fails, we might have duplicate processing. If this + # data was stored as custom metadata in S3, the upload operation would + # atomically set the flag. OTOH the frontend would have a harder time getting to it. + print(">> Updating image in DB") + db_conn.execute(""" + UPDATE + images + SET + is_watermarked = TRUE + WHERE + id = %s + """, [image_id]) + db_conn.commit() + +def process_assignment(assignment_id: int): + # Retrieve all UNPROCESSED images from DB. + # Some images may have already been processed by an earlier, partial job. + db_conn = connect_to_db() + images = db_conn.execute(""" + SELECT + id, + s3_path + FROM + images + WHERE + assignment_id = %s AND is_watermarked = false + """, [assignment_id]).fetchall() + + # Connect to S3. We will use this to client to download images. + # Just like app/src/lib/server/s3.ts, this would read from .env in prod. + s3_client = connect_to_s3() + for image_id, s3_key in images: + process_image(db_conn, s3_client, image_id, s3_key) + + # Assuming all of the above worked, we have finished this stage of processing assignments. + db_conn.execute(""" + UPDATE + assignments + SET + state = 'AWAITING_OWNER_NOTIFICATION' :: assignment_state + WHERE + id = %s + """, [assignment_id]) + db_conn.commit() + +beanstalk = connect_to_beanstalkd() +beanstalk.watch("watermarking") + +while True: + job = beanstalk.reserve() + if not job: + continue + print(f"!! Got job: {job.body}") + + job_body = json.loads(job.body) + process_assignment(job_body["assignmentId"]) + + job.delete() |