✂️ Implement message chunking mechanism for NIP-44 size limit compliance
This ensures all messages can be properly encrypted and transmitted regardless of size. Fixes issue #2
This commit is contained in:
parent
a4134fa416
commit
89d9dc3cbe
GPG key ID:
7CC8056A5A04557E
4 changed files with 246 additions and 65 deletions
24
consts.ts
24
consts.ts
|
|
@ -18,3 +18,27 @@ export const MIN_POW = 8;
|
|||
* - Difficulty 21: ~5-6 seconds
|
||||
*/
|
||||
export const POW_TO_MINE = 10;
|
||||
|
||||
/**
|
||||
* Maximum size of a note chunk in bytes.
|
||||
*
|
||||
* This value determines the maximum size of a note that can be encrypted and
|
||||
* sent in a single chunk.
|
||||
*/
|
||||
export const MAX_CHUNK_SIZE = 32768;
|
||||
|
||||
/**
|
||||
* Interval for cleaning up expired note chunks in milliseconds.
|
||||
*
|
||||
* This value determines how often the relay will check for and remove expired
|
||||
* note chunks from the database.
|
||||
*/
|
||||
export const CHUNK_CLEANUP_INTERVAL = 1000 * 60 * 60;
|
||||
|
||||
/**
|
||||
* Maximum age of a note chunk in milliseconds.
|
||||
*
|
||||
* This value determines the maximum duration a note chunk can remain in the
|
||||
* database before it is considered expired and eligible for cleanup.
|
||||
*/
|
||||
export const CHUNK_MAX_AGE = 1000 * 60 * 60 * 24;
|
||||
|
|
|
|||
174
eventEncryptionDecryption.ts
Normal file
174
eventEncryptionDecryption.ts
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
import type { Database } from '@db/sqlite';
|
||||
import * as nostrTools from '@nostr/tools';
|
||||
import { nip44 } from '@nostr/tools';
|
||||
import { MAX_CHUNK_SIZE, MIN_POW, POW_TO_MINE } from './consts.ts';
|
||||
import {
|
||||
getCCNPrivateKey,
|
||||
getCCNPubkey,
|
||||
randomTimeUpTo2DaysInThePast,
|
||||
} from './utils.ts';
|
||||
import { sql } from './utils/queries.ts';
|
||||
|
||||
export class EventAlreadyExistsException extends Error {}
|
||||
export class ChunkedEventReceived extends Error {}
|
||||
|
||||
export async function createEncryptedEvent(
|
||||
event: nostrTools.VerifiedEvent,
|
||||
): Promise<nostrTools.VerifiedEvent | nostrTools.VerifiedEvent[]> {
|
||||
if (!event.id) throw new Error('Event must have an ID');
|
||||
if (!event.sig) throw new Error('Event must be signed');
|
||||
|
||||
const ccnPubKey = await getCCNPubkey();
|
||||
const ccnPrivateKey = await getCCNPrivateKey();
|
||||
|
||||
const eventJson = JSON.stringify(event);
|
||||
if (eventJson.length <= MAX_CHUNK_SIZE) {
|
||||
const randomPrivateKey = nostrTools.generateSecretKey();
|
||||
const randomPrivateKeyPubKey = nostrTools.getPublicKey(randomPrivateKey);
|
||||
const conversationKey = nip44.getConversationKey(
|
||||
randomPrivateKey,
|
||||
ccnPubKey,
|
||||
);
|
||||
const sealTemplate = {
|
||||
kind: 13,
|
||||
created_at: randomTimeUpTo2DaysInThePast(),
|
||||
content: nip44.encrypt(eventJson, conversationKey),
|
||||
tags: [],
|
||||
};
|
||||
const seal = nostrTools.finalizeEvent(sealTemplate, ccnPrivateKey);
|
||||
const giftWrapTemplate = {
|
||||
kind: 1059,
|
||||
created_at: randomTimeUpTo2DaysInThePast(),
|
||||
content: nip44.encrypt(JSON.stringify(seal), conversationKey),
|
||||
tags: [['p', ccnPubKey]],
|
||||
pubkey: randomPrivateKeyPubKey,
|
||||
};
|
||||
const minedGiftWrap = nostrTools.nip13.minePow(
|
||||
giftWrapTemplate,
|
||||
POW_TO_MINE,
|
||||
);
|
||||
const giftWrap = nostrTools.finalizeEvent(minedGiftWrap, randomPrivateKey);
|
||||
return giftWrap;
|
||||
}
|
||||
|
||||
const chunks: string[] = [];
|
||||
for (let i = 0; i < eventJson.length; i += MAX_CHUNK_SIZE)
|
||||
chunks.push(eventJson.slice(i, i + MAX_CHUNK_SIZE));
|
||||
|
||||
const messageId = crypto.randomUUID();
|
||||
const totalChunks = chunks.length;
|
||||
|
||||
const encryptedChunks = [];
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
const randomPrivateKey = nostrTools.generateSecretKey();
|
||||
const randomPrivateKeyPubKey = nostrTools.getPublicKey(randomPrivateKey);
|
||||
const conversationKey = nip44.getConversationKey(
|
||||
randomPrivateKey,
|
||||
ccnPubKey,
|
||||
);
|
||||
|
||||
const sealTemplate = {
|
||||
kind: 13,
|
||||
created_at: randomTimeUpTo2DaysInThePast(),
|
||||
content: nip44.encrypt(chunk, conversationKey),
|
||||
tags: [['chunk', String(i), String(totalChunks), messageId]],
|
||||
};
|
||||
|
||||
const seal = nostrTools.finalizeEvent(sealTemplate, ccnPrivateKey);
|
||||
const giftWrapTemplate = {
|
||||
kind: 1059,
|
||||
created_at: randomTimeUpTo2DaysInThePast(),
|
||||
content: nip44.encrypt(JSON.stringify(seal), conversationKey),
|
||||
tags: [['p', ccnPubKey]],
|
||||
pubkey: randomPrivateKeyPubKey,
|
||||
};
|
||||
|
||||
const minedGiftWrap = nostrTools.nip13.minePow(
|
||||
giftWrapTemplate,
|
||||
POW_TO_MINE,
|
||||
);
|
||||
encryptedChunks.push(
|
||||
nostrTools.finalizeEvent(minedGiftWrap, randomPrivateKey),
|
||||
);
|
||||
}
|
||||
|
||||
return encryptedChunks;
|
||||
}
|
||||
export async function decryptEvent(
|
||||
db: Database,
|
||||
event: nostrTools.Event,
|
||||
): Promise<nostrTools.VerifiedEvent> {
|
||||
const ccnPrivkey = await getCCNPrivateKey();
|
||||
|
||||
if (event.kind !== 1059) {
|
||||
throw new Error('Cannot decrypt event -- not a gift wrap');
|
||||
}
|
||||
|
||||
const pow = nostrTools.nip13.getPow(event.id);
|
||||
|
||||
if (pow < MIN_POW) {
|
||||
throw new Error('Cannot decrypt event -- PoW too low');
|
||||
}
|
||||
|
||||
const conversationKey = nip44.getConversationKey(ccnPrivkey, event.pubkey);
|
||||
const seal = JSON.parse(nip44.decrypt(event.content, conversationKey));
|
||||
if (!seal) throw new Error('Cannot decrypt event -- no seal');
|
||||
if (seal.kind !== 13) {
|
||||
throw new Error('Cannot decrypt event subevent -- not a seal');
|
||||
}
|
||||
|
||||
const chunkTag = seal.tags.find((tag: string[]) => tag[0] === 'chunk');
|
||||
if (!chunkTag) {
|
||||
const content = JSON.parse(nip44.decrypt(seal.content, conversationKey));
|
||||
return content as nostrTools.VerifiedEvent;
|
||||
}
|
||||
|
||||
const [_, chunkIndex, totalChunks, messageId] = chunkTag;
|
||||
const chunk = nip44.decrypt(seal.content, conversationKey);
|
||||
|
||||
try {
|
||||
sql`
|
||||
INSERT INTO event_chunks (
|
||||
message_id,
|
||||
chunk_index,
|
||||
total_chunks,
|
||||
chunk_data,
|
||||
conversation_key,
|
||||
created_at
|
||||
) VALUES (
|
||||
${messageId},
|
||||
${Number(chunkIndex)},
|
||||
${Number(totalChunks)},
|
||||
${chunk},
|
||||
${conversationKey},
|
||||
${Math.floor(Date.now() / 1000)}
|
||||
)
|
||||
`(db);
|
||||
|
||||
const chunks = sql`
|
||||
SELECT chunk_data
|
||||
FROM event_chunks
|
||||
WHERE message_id = ${messageId}
|
||||
ORDER BY chunk_index ASC
|
||||
`(db);
|
||||
|
||||
if (chunks.length === Number(totalChunks)) {
|
||||
const completeEventJson = chunks.map((c) => c.chunk_data).join('');
|
||||
|
||||
sql`DELETE FROM event_chunks WHERE message_id = ${messageId}`(db);
|
||||
|
||||
return JSON.parse(completeEventJson) as nostrTools.VerifiedEvent;
|
||||
}
|
||||
|
||||
throw new ChunkedEventReceived(
|
||||
`Chunked event received (${chunks.length}/${totalChunks}) - messageId: ${messageId}`,
|
||||
);
|
||||
} catch (e) {
|
||||
if (e instanceof Error && e.message.includes('UNIQUE constraint failed'))
|
||||
throw new Error(
|
||||
`Duplicate chunk received (${Number(chunkIndex) + 1}/${totalChunks}) - messageId: ${messageId}`,
|
||||
);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
100
index.ts
100
index.ts
|
|
@ -1,3 +1,5 @@
|
|||
import { randomBytes } from '@noble/ciphers/webcrypto';
|
||||
import * as nostrTools from '@nostr/tools';
|
||||
import { Database } from 'jsr:@db/sqlite';
|
||||
import { NSchema as n } from 'jsr:@nostrify/nostrify';
|
||||
import type {
|
||||
|
|
@ -6,10 +8,13 @@ import type {
|
|||
NostrFilter,
|
||||
} from 'jsr:@nostrify/types';
|
||||
import { encodeBase64 } from 'jsr:@std/encoding@0.224/base64';
|
||||
import { randomBytes } from '@noble/ciphers/webcrypto';
|
||||
import * as nostrTools from '@nostr/tools';
|
||||
import { nip44 } from '@nostr/tools';
|
||||
import { MIN_POW, POW_TO_MINE } from './consts.ts';
|
||||
import { CHUNK_CLEANUP_INTERVAL, CHUNK_MAX_AGE } from './consts.ts';
|
||||
import {
|
||||
ChunkedEventReceived,
|
||||
EventAlreadyExistsException,
|
||||
createEncryptedEvent,
|
||||
decryptEvent,
|
||||
} from './eventEncryptionDecryption.ts';
|
||||
import {
|
||||
getCCNPrivateKey,
|
||||
getCCNPubkey,
|
||||
|
|
@ -20,7 +25,6 @@ import {
|
|||
isReplaceableEvent,
|
||||
isValidJSON,
|
||||
parseATagQuery,
|
||||
randomTimeUpTo2DaysInThePast,
|
||||
} from './utils.ts';
|
||||
import { getEveFilePath } from './utils/files.ts';
|
||||
import { log, setupLogger } from './utils/logs.ts';
|
||||
|
|
@ -103,62 +107,6 @@ export function runMigrations(db: Database, latestVersion: number) {
|
|||
}
|
||||
}
|
||||
|
||||
async function createEncryptedEvent(
|
||||
event: nostrTools.VerifiedEvent,
|
||||
): Promise<nostrTools.VerifiedEvent> {
|
||||
if (!event.id) throw new Error('Event must have an ID');
|
||||
if (!event.sig) throw new Error('Event must be signed');
|
||||
const ccnPubKey = await getCCNPubkey();
|
||||
const ccnPrivateKey = await getCCNPrivateKey();
|
||||
const randomPrivateKey = nostrTools.generateSecretKey();
|
||||
const randomPrivateKeyPubKey = nostrTools.getPublicKey(randomPrivateKey);
|
||||
const conversationKey = nip44.getConversationKey(randomPrivateKey, ccnPubKey);
|
||||
const sealTemplate = {
|
||||
kind: 13,
|
||||
created_at: randomTimeUpTo2DaysInThePast(),
|
||||
content: nip44.encrypt(JSON.stringify(event), conversationKey),
|
||||
tags: [],
|
||||
};
|
||||
const seal = nostrTools.finalizeEvent(sealTemplate, ccnPrivateKey);
|
||||
const giftWrapTemplate = {
|
||||
kind: 1059,
|
||||
created_at: randomTimeUpTo2DaysInThePast(),
|
||||
content: nip44.encrypt(JSON.stringify(seal), conversationKey),
|
||||
tags: [['p', ccnPubKey]],
|
||||
pubkey: randomPrivateKeyPubKey,
|
||||
};
|
||||
const minedGiftWrap = nostrTools.nip13.minePow(giftWrapTemplate, POW_TO_MINE);
|
||||
const giftWrap = nostrTools.finalizeEvent(minedGiftWrap, randomPrivateKey);
|
||||
return giftWrap;
|
||||
}
|
||||
|
||||
async function decryptEvent(
|
||||
event: nostrTools.Event,
|
||||
): Promise<nostrTools.VerifiedEvent> {
|
||||
const ccnPrivkey = await getCCNPrivateKey();
|
||||
|
||||
if (event.kind !== 1059) {
|
||||
throw new Error('Cannot decrypt event -- not a gift wrap');
|
||||
}
|
||||
|
||||
const pow = nostrTools.nip13.getPow(event.id);
|
||||
|
||||
if (pow < MIN_POW) {
|
||||
throw new Error('Cannot decrypt event -- PoW too low');
|
||||
}
|
||||
|
||||
const conversationKey = nip44.getConversationKey(ccnPrivkey, event.pubkey);
|
||||
const seal = JSON.parse(nip44.decrypt(event.content, conversationKey));
|
||||
if (!seal) throw new Error('Cannot decrypt event -- no seal');
|
||||
if (seal.kind !== 13) {
|
||||
throw new Error('Cannot decrypt event subevent -- not a seal');
|
||||
}
|
||||
const content = JSON.parse(nip44.decrypt(seal.content, conversationKey));
|
||||
return content as nostrTools.VerifiedEvent;
|
||||
}
|
||||
|
||||
class EventAlreadyExistsException extends Error {}
|
||||
|
||||
function addEventToDb(
|
||||
decryptedEvent: nostrTools.VerifiedEvent,
|
||||
encryptedEvent: nostrTools.VerifiedEvent,
|
||||
|
|
@ -267,6 +215,11 @@ function encryptedEventIsInDb(event: nostrTools.VerifiedEvent) {
|
|||
`(db)[0];
|
||||
}
|
||||
|
||||
function cleanupOldChunks() {
|
||||
const cutoffTime = Math.floor((Date.now() - CHUNK_MAX_AGE) / 1000);
|
||||
sql`DELETE FROM event_chunks WHERE created_at < ${cutoffTime}`(db);
|
||||
}
|
||||
|
||||
async function setupAndSubscribeToExternalEvents() {
|
||||
const ccnPubkey = await getCCNPubkey();
|
||||
|
||||
|
|
@ -303,11 +256,14 @@ async function setupAndSubscribeToExternalEvents() {
|
|||
return;
|
||||
}
|
||||
if (encryptedEventIsInDb(event)) return;
|
||||
const decryptedEvent = await decryptEvent(event);
|
||||
try {
|
||||
const decryptedEvent = await decryptEvent(db, event);
|
||||
addEventToDb(decryptedEvent, event);
|
||||
} catch (e) {
|
||||
if (e instanceof EventAlreadyExistsException) return;
|
||||
if (e instanceof ChunkedEventReceived) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
|
|
@ -338,8 +294,15 @@ async function setupAndSubscribeToExternalEvents() {
|
|||
const encryptedCCNCreationEvent =
|
||||
await createEncryptedEvent(ccnCreationEvent);
|
||||
if (timerCleaned) return; // in case we get an event before the timer is cleaned
|
||||
await Promise.any(pool.publish(relays, encryptedCCNCreationEvent));
|
||||
if (Array.isArray(encryptedCCNCreationEvent)) {
|
||||
for (const event of encryptedCCNCreationEvent)
|
||||
await Promise.any(pool.publish(relays, event));
|
||||
} else {
|
||||
await Promise.any(pool.publish(relays, encryptedCCNCreationEvent));
|
||||
}
|
||||
}, 10000);
|
||||
|
||||
setInterval(cleanupOldChunks, CHUNK_CLEANUP_INTERVAL);
|
||||
}
|
||||
|
||||
await setupAndSubscribeToExternalEvents();
|
||||
|
|
@ -644,14 +607,21 @@ async function handleEvent(
|
|||
|
||||
const encryptedEvent = await createEncryptedEvent(event);
|
||||
try {
|
||||
addEventToDb(event, encryptedEvent);
|
||||
if (Array.isArray(encryptedEvent)) {
|
||||
await Promise.all(
|
||||
encryptedEvent.map((chunk) => Promise.any(pool.publish(relays, chunk))),
|
||||
);
|
||||
addEventToDb(event, encryptedEvent[0]);
|
||||
} else {
|
||||
addEventToDb(event, encryptedEvent);
|
||||
await Promise.any(pool.publish(relays, encryptedEvent));
|
||||
}
|
||||
} catch (e) {
|
||||
if (e instanceof EventAlreadyExistsException) {
|
||||
log.warn('Event already exists');
|
||||
return;
|
||||
}
|
||||
}
|
||||
await Promise.any(pool.publish(relays, encryptedEvent));
|
||||
|
||||
connection.socket.send(JSON.stringify(['OK', event.id, true, 'Event added']));
|
||||
|
||||
|
|
|
|||
13
migrations/4-createChunksStore.sql
Normal file
13
migrations/4-createChunksStore.sql
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
CREATE TABLE event_chunks (
|
||||
chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
message_id TEXT NOT NULL,
|
||||
chunk_index INTEGER NOT NULL,
|
||||
total_chunks INTEGER NOT NULL,
|
||||
chunk_data TEXT NOT NULL,
|
||||
conversation_key TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
UNIQUE(message_id, chunk_index)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_event_chunks_message_id ON event_chunks(message_id);
|
||||
CREATE INDEX idx_event_chunks_created_at ON event_chunks(created_at);
|
||||
Loading…
Add table
Reference in a new issue