diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5ad5c50..49c3309 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,6 +17,7 @@ on: push: branches: - main + - 20-feature-remove-expired-items-for-postgres-adapter env: REGISTRY: ghcr.io @@ -51,4 +52,4 @@ jobs: target: production push: true tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file + labels: ${{ steps.meta.outputs.labels }} diff --git a/README.md b/README.md index 6b18477..8243476 100644 --- a/README.md +++ b/README.md @@ -18,22 +18,25 @@ It use Keyv as a simple K/V store so you can use the database of your choice. ## Environement Variables -| Name | Description | Default value | -| --------------- | ------------------------------------------------------------ | ---------------- | -| `PORT` | Server listening port | 8080 | -| `GLOBAL_PREFIX` | API global prefix for every routes | `/api/v2` | -| `STORAGE_URI` | [Keyv](https://github.com/jaredwray/keyv) connection string, example: `redis://user:pass@localhost:6379`. Availabe Keyv storage adapter: redis, mongo, postgres and mysql | `""` (in memory **non-persistent**) | -| `STORAGE_TTL` | Time to live for data | null | -| `LOG_LEVEL` | Log level (`debug`, `verbose`, `log`, `warn`, `error`) | `warn` | -| `BODY_LIMIT` | Payload size limit for scenes or images | `50mb` | +| Name | Description | Default value | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| `PORT` | Server listening port | 8080 | +| `GLOBAL_PREFIX` | API global prefix for every routes | `/api/v2` | +| `STORAGE_URI` | [Keyv](https://github.com/jaredwray/keyv) connection string, example: `redis://user:pass@localhost:6379`. Availabe Keyv storage adapter: redis, mongo, postgres and mysql | `""` (in memory **non-persistent**) | +| `STORAGE_TTL` | Time to live for data | null | +| `LOG_LEVEL` | Log level (`debug`, `verbose`, `log`, `warn`, `error`) | `warn` | +| `BODY_LIMIT` | Payload size limit for scenes or images | `50mb` | +| `ENABLE_POSTGRES_TTL_SERVICE` | Enabling the Postgres TTL Service will clean up expired items once a day. This will break if used with a non-postgres `STORAGE_URI`. | `false` | ### Env Variables for Postgres + For setting postgres pool variables, a few code adjustment have to be made in the `storage.service.ts`: + ```typescript const store = new (require('@keyv/postgres'))({ uri, - max: 1 -}) + max: 1, +}); const keyv = new Keyv({ store, diff --git a/package-lock.json b/package-lock.json index 4398548..e70e21d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,13 +11,15 @@ "dependencies": { "@keyv/postgres": "^1.4.10", "@keyv/redis": "^2.8.1", - "@nestjs/common": "^10.3.0", - "@nestjs/core": "^10.3.0", - "@nestjs/platform-express": "^10.3.0", + "@nestjs/common": "^10.3.7", + "@nestjs/core": "^10.3.7", + "@nestjs/platform-express": "^10.3.7", + "@nestjs/schedule": "^4.0.1", "@types/keyv": "^3.1.4", "keyv": "^4.5.4", "nanoid": "^3.3.6", "npm-check-updates": "^16.14.12", + "pg": "^8.11.5", "reflect-metadata": "^0.1.12", "rimraf": "^5.0.5", "rxjs": "^7.8.1" @@ -25,7 +27,7 @@ "devDependencies": { "@nestjs/cli": "^10.2.1", "@nestjs/schematics": "^10.0.3", - "@nestjs/testing": "^10.3.0", + "@nestjs/testing": "^10.3.7", "@types/express": "^4.17.21", "@types/jest": "^29.5.11", "@types/node": "^20.10.6", @@ -1551,6 +1553,34 @@ "node": ">= 14" } }, + "node_modules/@keyv/postgres/node_modules/pg": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.6.2", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + }, + "engines": { + "node": ">= 8.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.1.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, "node_modules/@keyv/redis": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/@keyv/redis/-/redis-2.8.1.tgz", @@ -1739,9 +1769,9 @@ } }, "node_modules/@nestjs/common": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-10.3.0.tgz", - "integrity": "sha512-DGv34UHsZBxCM3H5QGE2XE/+oLJzz5+714JQjBhjD9VccFlQs3LRxo/epso4l7nJIiNlZkPyIUC8WzfU/5RTsQ==", + "version": "10.3.7", + "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-10.3.7.tgz", + "integrity": "sha512-gKFtFzcJznrwsRYjtNZoPAvSOPYdNgxbTYoAyLTpoy393cIKgLmJTHu6ReH8/qIB9AaZLdGaFLkx98W/tFWFUw==", "dependencies": { "iterare": "1.2.1", "tslib": "2.6.2", @@ -1754,7 +1784,7 @@ "peerDependencies": { "class-transformer": "*", "class-validator": "*", - "reflect-metadata": "^0.1.12", + "reflect-metadata": "^0.1.12 || ^0.2.0", "rxjs": "^7.1.0" }, "peerDependenciesMeta": { @@ -1767,9 +1797,9 @@ } }, "node_modules/@nestjs/core": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-10.3.0.tgz", - "integrity": "sha512-N06P5ncknW/Pm8bj964WvLIZn2gNhHliCBoAO1LeBvNImYkecqKcrmLbY49Fa1rmMfEM3MuBHeDys3edeuYAOA==", + "version": "10.3.7", + "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-10.3.7.tgz", + "integrity": "sha512-hsdlnfiQ3kgqHL5k7js3CU0PV7hBJVi+LfFMgCkoagRxNMf67z0GFGeOV2jk5d65ssB19qdYsDa1MGVuEaoUpg==", "hasInstallScript": true, "dependencies": { "@nuxtjs/opencollective": "0.3.2", @@ -1788,7 +1818,7 @@ "@nestjs/microservices": "^10.0.0", "@nestjs/platform-express": "^10.0.0", "@nestjs/websockets": "^10.0.0", - "reflect-metadata": "^0.1.12", + "reflect-metadata": "^0.1.12 || ^0.2.0", "rxjs": "^7.1.0" }, "peerDependenciesMeta": { @@ -1804,13 +1834,13 @@ } }, "node_modules/@nestjs/platform-express": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.3.0.tgz", - "integrity": "sha512-E4hUW48bYv8OHbP9XQg6deefmXb0pDSSuE38SdhA0mJ37zGY7C5EqqBUdlQk4ttfD+OdnbIgJ1zOokT6dd2d7A==", + "version": "10.3.7", + "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.3.7.tgz", + "integrity": "sha512-noNJ+PyIxQJLCKfuXz0tcQtlVAynfLIuKy62g70lEZ86UrIqSrZFqvWs/rFUgkbT6J8H7Rmv11hASOnX+7M2rA==", "dependencies": { "body-parser": "1.20.2", "cors": "2.8.5", - "express": "4.18.2", + "express": "4.19.2", "multer": "1.4.4-lts.1", "tslib": "2.6.2" }, @@ -1823,6 +1853,19 @@ "@nestjs/core": "^10.0.0" } }, + "node_modules/@nestjs/schedule": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.0.1.tgz", + "integrity": "sha512-cz2FNjsuoma+aGsG0cMmG6Dqg/BezbBWet1UTHtAuu6d2mXNTVcmoEQM2DIVG5Lfwb2hfSE2yZt8Moww+7y+mA==", + "dependencies": { + "cron": "3.1.6", + "uuid": "9.0.1" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" + } + }, "node_modules/@nestjs/schematics": { "version": "10.0.3", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-10.0.3.tgz", @@ -1840,9 +1883,9 @@ } }, "node_modules/@nestjs/testing": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@nestjs/testing/-/testing-10.3.0.tgz", - "integrity": "sha512-8DM+bw1qASCvaEnoHUQhypCOf54+G5R21MeFBMvnSk5DtKaWVZuzDP2GjLeYCpTH19WeP6LrrjHv3rX2LKU02A==", + "version": "10.3.7", + "resolved": "https://registry.npmjs.org/@nestjs/testing/-/testing-10.3.7.tgz", + "integrity": "sha512-PmwZXyoCC/m3F3IFgpgD+SNN6cDPQa/vi3YQxFruvfX3cuHq+P6ZFvBB7hwaKKsLlhA0so42LsMm41oFBkdouw==", "dev": true, "dependencies": { "tslib": "2.6.2" @@ -2473,6 +2516,11 @@ "@types/node": "*" } }, + "node_modules/@types/luxon": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.3.8.tgz", + "integrity": "sha512-jYvz8UMLDgy3a5SkGJne8H7VA7zPV2Lwohjx0V8V31+SqAjNmurWMkk9cQhfvlcnXWudBpK9xPM1n4rljOcHYQ==" + }, "node_modules/@types/methods": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz", @@ -4271,9 +4319,9 @@ "dev": true }, "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", "engines": { "node": ">= 0.6" } @@ -4359,6 +4407,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cron": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.6.tgz", + "integrity": "sha512-cvFiQCeVzsA+QPM6fhjBtlKGij7tLLISnTSvFxVdnFGLdz+ZdXN37kNe0i2gefmdD17XuZA6n2uPVwzl4FxW/w==", + "dependencies": { + "@types/luxon": "~3.3.0", + "luxon": "~3.4.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -5112,16 +5169,16 @@ "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==" }, "node_modules/express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -5152,29 +5209,6 @@ "node": ">= 0.10.0" } }, - "node_modules/express/node_modules/body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", - "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.1", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/express/node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -5193,20 +5227,6 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, - "node_modules/express/node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", - "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/express/node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -6221,9 +6241,9 @@ } }, "node_modules/ip": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", - "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.1.tgz", + "integrity": "sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==" }, "node_modules/ipaddr.js": { "version": "1.9.1", @@ -7348,6 +7368,14 @@ "node": ">=10" } }, + "node_modules/luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "engines": { + "node": ">=12" + } + }, "node_modules/macos-release": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz", @@ -8815,15 +8843,13 @@ } }, "node_modules/pg": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", - "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "version": "8.11.5", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.5.tgz", + "integrity": "sha512-jqgNHSKL5cbDjFlHyYsCXmQDrfIX/3RsNwYqpd4N0Kt8niLuNoRNH+aazv6cOd43gPh9Y4DjQCtb+X0MH0Hvnw==", "dependencies": { - "buffer-writer": "2.0.0", - "packet-reader": "1.0.0", - "pg-connection-string": "^2.6.2", - "pg-pool": "^3.6.1", - "pg-protocol": "^1.6.0", + "pg-connection-string": "^2.6.4", + "pg-pool": "^3.6.2", + "pg-protocol": "^1.6.1", "pg-types": "^2.1.0", "pgpass": "1.x" }, @@ -8849,9 +8875,9 @@ "optional": true }, "node_modules/pg-connection-string": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", - "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", + "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" }, "node_modules/pg-int8": { "version": "1.0.1", @@ -8862,17 +8888,17 @@ } }, "node_modules/pg-pool": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", - "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", + "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", "peerDependencies": { "pg": ">=8.0" } }, "node_modules/pg-protocol": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", - "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", + "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" }, "node_modules/pg-types": { "version": "2.2.0", @@ -11033,6 +11059,18 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -12597,6 +12635,23 @@ "integrity": "sha512-iSAox9VrvpXqQvhjq6RlO+A2YdDC+5n2LIfYaVKF/K7Pu3GMwDSsIS/H/9LtVeh2LIjfYiXkPwHR6m2GUWJ78A==", "requires": { "pg": "8.11.3" + }, + "dependencies": { + "pg": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-cloudflare": "^1.1.1", + "pg-connection-string": "^2.6.2", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + } + } } }, "@keyv/redis": { @@ -12721,9 +12776,9 @@ } }, "@nestjs/common": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-10.3.0.tgz", - "integrity": "sha512-DGv34UHsZBxCM3H5QGE2XE/+oLJzz5+714JQjBhjD9VccFlQs3LRxo/epso4l7nJIiNlZkPyIUC8WzfU/5RTsQ==", + "version": "10.3.7", + "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-10.3.7.tgz", + "integrity": "sha512-gKFtFzcJznrwsRYjtNZoPAvSOPYdNgxbTYoAyLTpoy393cIKgLmJTHu6ReH8/qIB9AaZLdGaFLkx98W/tFWFUw==", "requires": { "iterare": "1.2.1", "tslib": "2.6.2", @@ -12731,9 +12786,9 @@ } }, "@nestjs/core": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-10.3.0.tgz", - "integrity": "sha512-N06P5ncknW/Pm8bj964WvLIZn2gNhHliCBoAO1LeBvNImYkecqKcrmLbY49Fa1rmMfEM3MuBHeDys3edeuYAOA==", + "version": "10.3.7", + "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-10.3.7.tgz", + "integrity": "sha512-hsdlnfiQ3kgqHL5k7js3CU0PV7hBJVi+LfFMgCkoagRxNMf67z0GFGeOV2jk5d65ssB19qdYsDa1MGVuEaoUpg==", "requires": { "@nuxtjs/opencollective": "0.3.2", "fast-safe-stringify": "2.1.1", @@ -12744,17 +12799,26 @@ } }, "@nestjs/platform-express": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.3.0.tgz", - "integrity": "sha512-E4hUW48bYv8OHbP9XQg6deefmXb0pDSSuE38SdhA0mJ37zGY7C5EqqBUdlQk4ttfD+OdnbIgJ1zOokT6dd2d7A==", + "version": "10.3.7", + "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.3.7.tgz", + "integrity": "sha512-noNJ+PyIxQJLCKfuXz0tcQtlVAynfLIuKy62g70lEZ86UrIqSrZFqvWs/rFUgkbT6J8H7Rmv11hASOnX+7M2rA==", "requires": { "body-parser": "1.20.2", "cors": "2.8.5", - "express": "4.18.2", + "express": "4.19.2", "multer": "1.4.4-lts.1", "tslib": "2.6.2" } }, + "@nestjs/schedule": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.0.1.tgz", + "integrity": "sha512-cz2FNjsuoma+aGsG0cMmG6Dqg/BezbBWet1UTHtAuu6d2mXNTVcmoEQM2DIVG5Lfwb2hfSE2yZt8Moww+7y+mA==", + "requires": { + "cron": "3.1.6", + "uuid": "9.0.1" + } + }, "@nestjs/schematics": { "version": "10.0.3", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-10.0.3.tgz", @@ -12769,9 +12833,9 @@ } }, "@nestjs/testing": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@nestjs/testing/-/testing-10.3.0.tgz", - "integrity": "sha512-8DM+bw1qASCvaEnoHUQhypCOf54+G5R21MeFBMvnSk5DtKaWVZuzDP2GjLeYCpTH19WeP6LrrjHv3rX2LKU02A==", + "version": "10.3.7", + "resolved": "https://registry.npmjs.org/@nestjs/testing/-/testing-10.3.7.tgz", + "integrity": "sha512-PmwZXyoCC/m3F3IFgpgD+SNN6cDPQa/vi3YQxFruvfX3cuHq+P6ZFvBB7hwaKKsLlhA0so42LsMm41oFBkdouw==", "dev": true, "requires": { "tslib": "2.6.2" @@ -13271,6 +13335,11 @@ "@types/node": "*" } }, + "@types/luxon": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.3.8.tgz", + "integrity": "sha512-jYvz8UMLDgy3a5SkGJne8H7VA7zPV2Lwohjx0V8V31+SqAjNmurWMkk9cQhfvlcnXWudBpK9xPM1n4rljOcHYQ==" + }, "@types/methods": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz", @@ -14596,9 +14665,9 @@ "dev": true }, "cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" }, "cookie-signature": { "version": "1.0.6", @@ -14658,6 +14727,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "cron": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.6.tgz", + "integrity": "sha512-cvFiQCeVzsA+QPM6fhjBtlKGij7tLLISnTSvFxVdnFGLdz+ZdXN37kNe0i2gefmdD17XuZA6n2uPVwzl4FxW/w==", + "requires": { + "@types/luxon": "~3.3.0", + "luxon": "~3.4.0" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -15191,16 +15269,16 @@ "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==" }, "express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -15228,25 +15306,6 @@ "vary": "~1.1.2" }, "dependencies": { - "body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", - "requires": { - "bytes": "3.1.2", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.1", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - } - }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -15265,17 +15324,6 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, - "raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", - "requires": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - } - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -16012,9 +16060,9 @@ } }, "ip": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", - "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.1.tgz", + "integrity": "sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==" }, "ipaddr.js": { "version": "1.9.1", @@ -16860,6 +16908,11 @@ "yallist": "^4.0.0" } }, + "luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==" + }, "macos-release": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz", @@ -17939,16 +17992,14 @@ "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==" }, "pg": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", - "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "version": "8.11.5", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.5.tgz", + "integrity": "sha512-jqgNHSKL5cbDjFlHyYsCXmQDrfIX/3RsNwYqpd4N0Kt8niLuNoRNH+aazv6cOd43gPh9Y4DjQCtb+X0MH0Hvnw==", "requires": { - "buffer-writer": "2.0.0", - "packet-reader": "1.0.0", "pg-cloudflare": "^1.1.1", - "pg-connection-string": "^2.6.2", - "pg-pool": "^3.6.1", - "pg-protocol": "^1.6.0", + "pg-connection-string": "^2.6.4", + "pg-pool": "^3.6.2", + "pg-protocol": "^1.6.1", "pg-types": "^2.1.0", "pgpass": "1.x" } @@ -17960,9 +18011,9 @@ "optional": true }, "pg-connection-string": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", - "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", + "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" }, "pg-int8": { "version": "1.0.1", @@ -17970,15 +18021,15 @@ "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" }, "pg-pool": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", - "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", + "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", "requires": {} }, "pg-protocol": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", - "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", + "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" }, "pg-types": { "version": "2.2.0", @@ -19509,6 +19560,11 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/package.json b/package.json index 4a64101..85944ba 100644 --- a/package.json +++ b/package.json @@ -23,13 +23,15 @@ "dependencies": { "@keyv/postgres": "^1.4.10", "@keyv/redis": "^2.8.1", - "@nestjs/common": "^10.3.0", - "@nestjs/core": "^10.3.0", - "@nestjs/platform-express": "^10.3.0", + "@nestjs/common": "^10.3.7", + "@nestjs/core": "^10.3.7", + "@nestjs/platform-express": "^10.3.7", + "@nestjs/schedule": "^4.0.1", "@types/keyv": "^3.1.4", "keyv": "^4.5.4", "nanoid": "^3.3.6", "npm-check-updates": "^16.14.12", + "pg": "^8.11.5", "reflect-metadata": "^0.1.12", "rimraf": "^5.0.5", "rxjs": "^7.8.1" @@ -37,7 +39,7 @@ "devDependencies": { "@nestjs/cli": "^10.2.1", "@nestjs/schematics": "^10.0.3", - "@nestjs/testing": "^10.3.0", + "@nestjs/testing": "^10.3.7", "@types/express": "^4.17.21", "@types/jest": "^29.5.11", "@types/node": "^20.10.6", diff --git a/src/app.module.ts b/src/app.module.ts index 821d544..6c65b54 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,20 +1,40 @@ -import { MiddlewareConsumer, Module } from '@nestjs/common'; +import { Logger, MiddlewareConsumer, Module } from '@nestjs/common'; +import { ScheduleModule } from '@nestjs/schedule'; import { RawParserMiddleware } from './raw-parser.middleware'; import { ScenesController } from './scenes/scenes.controller'; import { StorageService } from './storage/storage.service'; import { RoomsController } from './rooms/rooms.controller'; import { FilesController } from './files/files.controller'; import { HealthController } from './health/health.controller'; +import { PostgresTtlService } from './ttl/postgres_ttl.service'; + +const logger = new Logger('AppModule'); + +const buildProviders = () => { + const ttlProvider = addTtlProvider(); + const providers: any[] = [StorageService]; + if (ttlProvider) { + providers.push(ttlProvider); + } + return providers; +}; + +const addTtlProvider = () => { + if (process.env['ENABLE_POSTGRES_TTL_SERVICE'] == 'true') { + logger.log('Enabling PostgresTtlService'); + return PostgresTtlService; + } +}; @Module({ - imports: [], + imports: [ScheduleModule.forRoot()], controllers: [ ScenesController, RoomsController, FilesController, HealthController, ], - providers: [StorageService], + providers: buildProviders(), }) export class AppModule { configure(consumer: MiddlewareConsumer) { diff --git a/src/ttl/postgres_ttl.service.spec.ts b/src/ttl/postgres_ttl.service.spec.ts new file mode 100644 index 0000000..ed3373c --- /dev/null +++ b/src/ttl/postgres_ttl.service.spec.ts @@ -0,0 +1,98 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { PostgresTtlService } from './postgres_ttl.service'; +import { StorageNamespace, StorageService } from '../storage/storage.service'; +import { + clearDatabase, + createTestDatabaseSetup, +} from '../../test/helpers/postgres_helper'; + +beforeAll(() => { + return createTestDatabaseSetup(); +}); + +describe('PostgresTtlService', () => { + let postgresTtlService: PostgresTtlService; + let storageService: StorageService; + + const setupServicesWithTtl = async (ttl: string) => { + process.env[`STORAGE_TTL`] = ttl; + const module: TestingModule = await Test.createTestingModule({ + providers: [PostgresTtlService, StorageService], + }).compile(); + storageService = module.get(StorageService); + postgresTtlService = module.get(PostgresTtlService); + }; + + afterEach(() => { + return clearDatabase(); + }); + + it('should be defined', async () => { + await setupServicesWithTtl('10'); + expect(postgresTtlService).toBeDefined(); + }); + + it('deletes items from postgres database which a ttl older than now', async () => { + await setupServicesWithTtl('-10000'); + await storageService.set('key', 'value', StorageNamespace.ROOMS); + const expired_items_count = await postgresTtlService.deleteExpiredItems(); + + expect(expired_items_count).toBe(1); + expect( + await storageService.get('key', StorageNamespace.ROOMS), + ).toBeUndefined(); + }); + + it('does not delete items from postgres database which a ttl newer than now', async () => { + await setupServicesWithTtl('1000'); + await storageService.set('key', 'value', StorageNamespace.ROOMS); + const expired_items_count = await postgresTtlService.deleteExpiredItems(); + + expect(expired_items_count).toBe(0); + expect(await storageService.get('key', StorageNamespace.ROOMS)).toEqual( + 'value', + ); + }); + + it('does not delete items after switching ttls', async () => { + await setupServicesWithTtl('1000'); + await storageService.set('new', 'new-value', StorageNamespace.ROOMS); + + await setupServicesWithTtl('-1000'); + await storageService.set( + 'expired', + 'expired-value', + StorageNamespace.ROOMS, + ); + + const expired_items_count = await postgresTtlService.deleteExpiredItems(); + + expect(expired_items_count).toBe(1); + expect(await storageService.get('new', StorageNamespace.ROOMS)).toEqual( + 'new-value', + ); + expect( + await storageService.get('expired', StorageNamespace.ROOMS), + ).toBeUndefined(); + }); + + it('deletes items from in all namespaces', async () => { + await setupServicesWithTtl('-10000'); + await storageService.set('key-rooms', 'value', StorageNamespace.ROOMS); + await storageService.set('key-files', 'value', StorageNamespace.FILES); + await storageService.set('key-scenes', 'value', StorageNamespace.SCENES); + + const expired_items_count = await postgresTtlService.deleteExpiredItems(); + + expect(expired_items_count).toBe(3); + expect( + await storageService.get('key-rooms', StorageNamespace.ROOMS), + ).toBeUndefined(); + expect( + await storageService.get('key-files', StorageNamespace.FILES), + ).toBeUndefined(); + expect( + await storageService.get('key-scenes', StorageNamespace.SCENES), + ).toBeUndefined(); + }); +}); diff --git a/src/ttl/postgres_ttl.service.ts b/src/ttl/postgres_ttl.service.ts new file mode 100644 index 0000000..22a1996 --- /dev/null +++ b/src/ttl/postgres_ttl.service.ts @@ -0,0 +1,46 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { Client } from 'pg'; + +@Injectable() +export class PostgresTtlService { + private readonly logger = new Logger(PostgresTtlService.name); + + @Cron(CronExpression.EVERY_DAY_AT_4AM) + async handleCron() { + this.logger.log('Starting PostgresTtlService to clean up expired data.'); + this.deleteExpiredItems(); + this.logger.log('Finished.'); + } + + async deleteExpiredItems() { + const uri: string = process.env[`STORAGE_URI`]; + if (!uri) { + this.logger.error(`STORAGE_URI is undefined, cannot clean up old items`); + return; + } + + // Set up client with uri: + const client = new Client({ connectionString: uri }); + let expired_items_count = 0; + + try { + // Connect to the database + client.connect(); + // Delete all expired items. TTL is stored in milliseconds: + const queryResult = await client.query( + "DELETE FROM keyv WHERE (keyv.value::json ->> 'expires')::bigint / 1000 <= extract(epoch from now());", + ); + + this.logger.log('Deleted expired items:', queryResult.rowCount); + expired_items_count = queryResult.rowCount; + } catch (error) { + this.logger.error('Error executing query:', error); + } finally { + // Alwys release the connection afterwards: + await client.end(); + } + + return expired_items_count; + } +} diff --git a/test/helpers/postgres_helper.ts b/test/helpers/postgres_helper.ts new file mode 100644 index 0000000..535a690 --- /dev/null +++ b/test/helpers/postgres_helper.ts @@ -0,0 +1,40 @@ +import { Logger } from '@nestjs/common'; +import { Client } from 'pg'; +const logger = new Logger(); + +// This is a very simple helper to make sure that the tests do not overwrite the local development database. +export const createTestDatabaseSetup = async () => { + const uri: string = process.env[`STORAGE_URI`]; + + // This is all very specific to the docker-compose.yml. We should refactor this later. This will break when query parameters are added: + const parts = uri.split('/'); + if (parts.length < 1) { + logger.error('No database match found'); + return; + } + const databaseSuffix = '-test'; + const database = parts[parts.length - 1] + databaseSuffix; + process.env['STORAGE_URI'] = process.env['STORAGE_URI'] + databaseSuffix; + + // Set up client with uri: + const client = new Client({ connectionString: uri }); + client.connect(); + + return client + .query(`CREATE DATABASE "${database}"`) + .catch((e) => logger.debug('Error executing query:', e)) // this will happen after the first run, so we'll discard it + .then(() => client.end()); +}; + +// This helper is required for testing the postgres ttl service. It is designed specificially to clean up after each postgres spec. +// Therefore, we don't want to include it in other specs. +export const clearDatabase = async () => { + const uri: string = process.env[`STORAGE_URI`]; + // Set up client with uri: + const client = new Client({ connectionString: uri }); + client.connect(); + return client + .query('DELETE FROM keyv;') + .catch((e) => console.error(e.stack)) + .then(() => client.end()); +};