From 89d9dc3cbeb0e7a1ee94ee42cff88e33debb994f Mon Sep 17 00:00:00 2001 From: Danny Morabito Date: Mon, 24 Mar 2025 20:14:52 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=82=EF=B8=8F=20Implement=20message=20chun?= =?UTF-8?q?king=20mechanism=20for=20NIP-44=20size=20limit=20compliance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This ensures all messages can be properly encrypted and transmitted regardless of size. Fixes issue #2 --- consts.ts | 24 ++++ eventEncryptionDecryption.ts | 174 +++++++++++++++++++++++++++++ index.ts | 100 ++++++----------- migrations/4-createChunksStore.sql | 13 +++ 4 files changed, 246 insertions(+), 65 deletions(-) create mode 100644 eventEncryptionDecryption.ts create mode 100644 migrations/4-createChunksStore.sql diff --git a/consts.ts b/consts.ts index cb2ce39..2b8235f 100644 --- a/consts.ts +++ b/consts.ts @@ -18,3 +18,27 @@ export const MIN_POW = 8; * - Difficulty 21: ~5-6 seconds */ export const POW_TO_MINE = 10; + +/** + * Maximum size of a note chunk in bytes. + * + * This value determines the maximum size of a note that can be encrypted and + * sent in a single chunk. + */ +export const MAX_CHUNK_SIZE = 32768; + +/** + * Interval for cleaning up expired note chunks in milliseconds. + * + * This value determines how often the relay will check for and remove expired + * note chunks from the database. + */ +export const CHUNK_CLEANUP_INTERVAL = 1000 * 60 * 60; + +/** + * Maximum age of a note chunk in milliseconds. + * + * This value determines the maximum duration a note chunk can remain in the + * database before it is considered expired and eligible for cleanup. + */ +export const CHUNK_MAX_AGE = 1000 * 60 * 60 * 24; diff --git a/eventEncryptionDecryption.ts b/eventEncryptionDecryption.ts new file mode 100644 index 0000000..d77e2f5 --- /dev/null +++ b/eventEncryptionDecryption.ts @@ -0,0 +1,174 @@ +import type { Database } from '@db/sqlite'; +import * as nostrTools from '@nostr/tools'; +import { nip44 } from '@nostr/tools'; +import { MAX_CHUNK_SIZE, MIN_POW, POW_TO_MINE } from './consts.ts'; +import { + getCCNPrivateKey, + getCCNPubkey, + randomTimeUpTo2DaysInThePast, +} from './utils.ts'; +import { sql } from './utils/queries.ts'; + +export class EventAlreadyExistsException extends Error {} +export class ChunkedEventReceived extends Error {} + +export async function createEncryptedEvent( + event: nostrTools.VerifiedEvent, +): Promise { + if (!event.id) throw new Error('Event must have an ID'); + if (!event.sig) throw new Error('Event must be signed'); + + const ccnPubKey = await getCCNPubkey(); + const ccnPrivateKey = await getCCNPrivateKey(); + + const eventJson = JSON.stringify(event); + if (eventJson.length <= MAX_CHUNK_SIZE) { + const randomPrivateKey = nostrTools.generateSecretKey(); + const randomPrivateKeyPubKey = nostrTools.getPublicKey(randomPrivateKey); + const conversationKey = nip44.getConversationKey( + randomPrivateKey, + ccnPubKey, + ); + const sealTemplate = { + kind: 13, + created_at: randomTimeUpTo2DaysInThePast(), + content: nip44.encrypt(eventJson, conversationKey), + tags: [], + }; + const seal = nostrTools.finalizeEvent(sealTemplate, ccnPrivateKey); + const giftWrapTemplate = { + kind: 1059, + created_at: randomTimeUpTo2DaysInThePast(), + content: nip44.encrypt(JSON.stringify(seal), conversationKey), + tags: [['p', ccnPubKey]], + pubkey: randomPrivateKeyPubKey, + }; + const minedGiftWrap = nostrTools.nip13.minePow( + giftWrapTemplate, + POW_TO_MINE, + ); + const giftWrap = nostrTools.finalizeEvent(minedGiftWrap, randomPrivateKey); + return giftWrap; + } + + const chunks: string[] = []; + for (let i = 0; i < eventJson.length; i += MAX_CHUNK_SIZE) + chunks.push(eventJson.slice(i, i + MAX_CHUNK_SIZE)); + + const messageId = crypto.randomUUID(); + const totalChunks = chunks.length; + + const encryptedChunks = []; + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]; + const randomPrivateKey = nostrTools.generateSecretKey(); + const randomPrivateKeyPubKey = nostrTools.getPublicKey(randomPrivateKey); + const conversationKey = nip44.getConversationKey( + randomPrivateKey, + ccnPubKey, + ); + + const sealTemplate = { + kind: 13, + created_at: randomTimeUpTo2DaysInThePast(), + content: nip44.encrypt(chunk, conversationKey), + tags: [['chunk', String(i), String(totalChunks), messageId]], + }; + + const seal = nostrTools.finalizeEvent(sealTemplate, ccnPrivateKey); + const giftWrapTemplate = { + kind: 1059, + created_at: randomTimeUpTo2DaysInThePast(), + content: nip44.encrypt(JSON.stringify(seal), conversationKey), + tags: [['p', ccnPubKey]], + pubkey: randomPrivateKeyPubKey, + }; + + const minedGiftWrap = nostrTools.nip13.minePow( + giftWrapTemplate, + POW_TO_MINE, + ); + encryptedChunks.push( + nostrTools.finalizeEvent(minedGiftWrap, randomPrivateKey), + ); + } + + return encryptedChunks; +} +export async function decryptEvent( + db: Database, + event: nostrTools.Event, +): Promise { + const ccnPrivkey = await getCCNPrivateKey(); + + if (event.kind !== 1059) { + throw new Error('Cannot decrypt event -- not a gift wrap'); + } + + const pow = nostrTools.nip13.getPow(event.id); + + if (pow < MIN_POW) { + throw new Error('Cannot decrypt event -- PoW too low'); + } + + const conversationKey = nip44.getConversationKey(ccnPrivkey, event.pubkey); + const seal = JSON.parse(nip44.decrypt(event.content, conversationKey)); + if (!seal) throw new Error('Cannot decrypt event -- no seal'); + if (seal.kind !== 13) { + throw new Error('Cannot decrypt event subevent -- not a seal'); + } + + const chunkTag = seal.tags.find((tag: string[]) => tag[0] === 'chunk'); + if (!chunkTag) { + const content = JSON.parse(nip44.decrypt(seal.content, conversationKey)); + return content as nostrTools.VerifiedEvent; + } + + const [_, chunkIndex, totalChunks, messageId] = chunkTag; + const chunk = nip44.decrypt(seal.content, conversationKey); + + try { + sql` + INSERT INTO event_chunks ( + message_id, + chunk_index, + total_chunks, + chunk_data, + conversation_key, + created_at + ) VALUES ( + ${messageId}, + ${Number(chunkIndex)}, + ${Number(totalChunks)}, + ${chunk}, + ${conversationKey}, + ${Math.floor(Date.now() / 1000)} + ) + `(db); + + const chunks = sql` + SELECT chunk_data + FROM event_chunks + WHERE message_id = ${messageId} + ORDER BY chunk_index ASC + `(db); + + if (chunks.length === Number(totalChunks)) { + const completeEventJson = chunks.map((c) => c.chunk_data).join(''); + + sql`DELETE FROM event_chunks WHERE message_id = ${messageId}`(db); + + return JSON.parse(completeEventJson) as nostrTools.VerifiedEvent; + } + + throw new ChunkedEventReceived( + `Chunked event received (${chunks.length}/${totalChunks}) - messageId: ${messageId}`, + ); + } catch (e) { + if (e instanceof Error && e.message.includes('UNIQUE constraint failed')) + throw new Error( + `Duplicate chunk received (${Number(chunkIndex) + 1}/${totalChunks}) - messageId: ${messageId}`, + ); + throw e; + } +} diff --git a/index.ts b/index.ts index b33b149..c3f110f 100644 --- a/index.ts +++ b/index.ts @@ -1,3 +1,5 @@ +import { randomBytes } from '@noble/ciphers/webcrypto'; +import * as nostrTools from '@nostr/tools'; import { Database } from 'jsr:@db/sqlite'; import { NSchema as n } from 'jsr:@nostrify/nostrify'; import type { @@ -6,10 +8,13 @@ import type { NostrFilter, } from 'jsr:@nostrify/types'; import { encodeBase64 } from 'jsr:@std/encoding@0.224/base64'; -import { randomBytes } from '@noble/ciphers/webcrypto'; -import * as nostrTools from '@nostr/tools'; -import { nip44 } from '@nostr/tools'; -import { MIN_POW, POW_TO_MINE } from './consts.ts'; +import { CHUNK_CLEANUP_INTERVAL, CHUNK_MAX_AGE } from './consts.ts'; +import { + ChunkedEventReceived, + EventAlreadyExistsException, + createEncryptedEvent, + decryptEvent, +} from './eventEncryptionDecryption.ts'; import { getCCNPrivateKey, getCCNPubkey, @@ -20,7 +25,6 @@ import { isReplaceableEvent, isValidJSON, parseATagQuery, - randomTimeUpTo2DaysInThePast, } from './utils.ts'; import { getEveFilePath } from './utils/files.ts'; import { log, setupLogger } from './utils/logs.ts'; @@ -103,62 +107,6 @@ export function runMigrations(db: Database, latestVersion: number) { } } -async function createEncryptedEvent( - event: nostrTools.VerifiedEvent, -): Promise { - if (!event.id) throw new Error('Event must have an ID'); - if (!event.sig) throw new Error('Event must be signed'); - const ccnPubKey = await getCCNPubkey(); - const ccnPrivateKey = await getCCNPrivateKey(); - const randomPrivateKey = nostrTools.generateSecretKey(); - const randomPrivateKeyPubKey = nostrTools.getPublicKey(randomPrivateKey); - const conversationKey = nip44.getConversationKey(randomPrivateKey, ccnPubKey); - const sealTemplate = { - kind: 13, - created_at: randomTimeUpTo2DaysInThePast(), - content: nip44.encrypt(JSON.stringify(event), conversationKey), - tags: [], - }; - const seal = nostrTools.finalizeEvent(sealTemplate, ccnPrivateKey); - const giftWrapTemplate = { - kind: 1059, - created_at: randomTimeUpTo2DaysInThePast(), - content: nip44.encrypt(JSON.stringify(seal), conversationKey), - tags: [['p', ccnPubKey]], - pubkey: randomPrivateKeyPubKey, - }; - const minedGiftWrap = nostrTools.nip13.minePow(giftWrapTemplate, POW_TO_MINE); - const giftWrap = nostrTools.finalizeEvent(minedGiftWrap, randomPrivateKey); - return giftWrap; -} - -async function decryptEvent( - event: nostrTools.Event, -): Promise { - const ccnPrivkey = await getCCNPrivateKey(); - - if (event.kind !== 1059) { - throw new Error('Cannot decrypt event -- not a gift wrap'); - } - - const pow = nostrTools.nip13.getPow(event.id); - - if (pow < MIN_POW) { - throw new Error('Cannot decrypt event -- PoW too low'); - } - - const conversationKey = nip44.getConversationKey(ccnPrivkey, event.pubkey); - const seal = JSON.parse(nip44.decrypt(event.content, conversationKey)); - if (!seal) throw new Error('Cannot decrypt event -- no seal'); - if (seal.kind !== 13) { - throw new Error('Cannot decrypt event subevent -- not a seal'); - } - const content = JSON.parse(nip44.decrypt(seal.content, conversationKey)); - return content as nostrTools.VerifiedEvent; -} - -class EventAlreadyExistsException extends Error {} - function addEventToDb( decryptedEvent: nostrTools.VerifiedEvent, encryptedEvent: nostrTools.VerifiedEvent, @@ -267,6 +215,11 @@ function encryptedEventIsInDb(event: nostrTools.VerifiedEvent) { `(db)[0]; } +function cleanupOldChunks() { + const cutoffTime = Math.floor((Date.now() - CHUNK_MAX_AGE) / 1000); + sql`DELETE FROM event_chunks WHERE created_at < ${cutoffTime}`(db); +} + async function setupAndSubscribeToExternalEvents() { const ccnPubkey = await getCCNPubkey(); @@ -303,11 +256,14 @@ async function setupAndSubscribeToExternalEvents() { return; } if (encryptedEventIsInDb(event)) return; - const decryptedEvent = await decryptEvent(event); try { + const decryptedEvent = await decryptEvent(db, event); addEventToDb(decryptedEvent, event); } catch (e) { if (e instanceof EventAlreadyExistsException) return; + if (e instanceof ChunkedEventReceived) { + return; + } } }, }, @@ -338,8 +294,15 @@ async function setupAndSubscribeToExternalEvents() { const encryptedCCNCreationEvent = await createEncryptedEvent(ccnCreationEvent); if (timerCleaned) return; // in case we get an event before the timer is cleaned - await Promise.any(pool.publish(relays, encryptedCCNCreationEvent)); + if (Array.isArray(encryptedCCNCreationEvent)) { + for (const event of encryptedCCNCreationEvent) + await Promise.any(pool.publish(relays, event)); + } else { + await Promise.any(pool.publish(relays, encryptedCCNCreationEvent)); + } }, 10000); + + setInterval(cleanupOldChunks, CHUNK_CLEANUP_INTERVAL); } await setupAndSubscribeToExternalEvents(); @@ -644,14 +607,21 @@ async function handleEvent( const encryptedEvent = await createEncryptedEvent(event); try { - addEventToDb(event, encryptedEvent); + if (Array.isArray(encryptedEvent)) { + await Promise.all( + encryptedEvent.map((chunk) => Promise.any(pool.publish(relays, chunk))), + ); + addEventToDb(event, encryptedEvent[0]); + } else { + addEventToDb(event, encryptedEvent); + await Promise.any(pool.publish(relays, encryptedEvent)); + } } catch (e) { if (e instanceof EventAlreadyExistsException) { log.warn('Event already exists'); return; } } - await Promise.any(pool.publish(relays, encryptedEvent)); connection.socket.send(JSON.stringify(['OK', event.id, true, 'Event added'])); diff --git a/migrations/4-createChunksStore.sql b/migrations/4-createChunksStore.sql new file mode 100644 index 0000000..d554cd4 --- /dev/null +++ b/migrations/4-createChunksStore.sql @@ -0,0 +1,13 @@ +CREATE TABLE event_chunks ( + chunk_id INTEGER PRIMARY KEY AUTOINCREMENT, + message_id TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + total_chunks INTEGER NOT NULL, + chunk_data TEXT NOT NULL, + conversation_key TEXT NOT NULL, + created_at INTEGER NOT NULL, + UNIQUE(message_id, chunk_index) +); + +CREATE INDEX idx_event_chunks_message_id ON event_chunks(message_id); +CREATE INDEX idx_event_chunks_created_at ON event_chunks(created_at); \ No newline at end of file