Merge pull request #142 from Askhz/feat/memory-cloud-sync

feat: add memory cloud sync to costrict-web API
This commit is contained in:
Askhz 2026-05-21 20:40:36 +08:00 committed by GitHub
commit 23711c45a8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 676 additions and 0 deletions

View File

@ -0,0 +1,283 @@
/**
* Memory Cloud Sync Service
*
* Syncs auto-memory files (personal memory) between the local filesystem
* and the costrict-web cloud API.
*
* API contract (costrict-web server/internal/memory):
* POST /api/memories create a new memory
* PUT /api/memories/:id update an existing memory
*
* Sync semantics:
* - Initial scan: on startup, scan all .md files, compare against local
* .cloud_sync_state.json, upload new/changed files.
* - Incremental: file watcher triggers re-scan of changed files.
* - Content MD5 comparison prevents redundant uploads.
* - Deletions do NOT propagate to the server.
*/
import { createHash } from 'crypto'
import { readdir, readFile, writeFile } from 'fs/promises'
import { basename, join } from 'path'
import { getAutoMemPath, isAutoMemoryEnabled } from '../../../memdir/paths.js'
import {
getCoStrictBaseURL,
} from '../../../costrict/provider/auth.js'
import { loadCoStrictCredentials } from '../../../costrict/provider/credentials.js'
import { parseFrontmatter } from '../../../utils/frontmatterParser.js'
import { getProjectRoot } from '../../../bootstrap/state.js'
import type {
CloudSyncState,
CreateMemoryRequest,
MemoryResponse,
MemoryUploadResult,
SyncedFileState,
UpdateMemoryRequest,
} from './types.js'
const API_TIMEOUT_MS = 30_000
// ─── Auth & endpoint ──────────────────────────────────────────
function getMemoryAPIEndpoint(path: string): string {
const baseUrl = getCoStrictBaseURL()
return `${baseUrl}/cloud-api/api/memories${path}`
}
async function getAuthHeaders(): Promise<{
headers?: Record<string, string>
error?: string
}> {
const creds = await loadCoStrictCredentials()
if (creds?.access_token) {
return {
headers: {
Authorization: `Bearer ${creds.access_token}`,
'Content-Type': 'application/json',
},
}
}
return { error: 'No CoStrict credentials available' }
}
/**
* Check if cloud sync can proceed (feature gating is done in watcher).
*/
export function isMemoryCloudSyncAvailable(): boolean {
return isAutoMemoryEnabled()
}
// ─── State file ────────────────────────────────────────────────
const STATE_FILENAME = '.cloud_sync_state.json'
function getStateFilePath(): string {
return join(getAutoMemPath(), STATE_FILENAME)
}
export async function loadSyncState(): Promise<CloudSyncState> {
try {
const raw = await readFile(getStateFilePath(), 'utf-8')
const parsed = JSON.parse(raw)
return { items: parsed.items ?? {} }
} catch {
return { items: {} }
}
}
export async function saveSyncState(state: CloudSyncState): Promise<void> {
await writeFile(
getStateFilePath(),
JSON.stringify(state, null, 2) + '\n',
'utf-8',
)
}
// ─── Content hash ──────────────────────────────────────────────
export function computeMD5(content: string): string {
return createHash('md5').update(content, 'utf8').digest('hex')
}
// ─── Upload ────────────────────────────────────────────────────
/**
* Upload a single memory file to the cloud.
* Uses POST for new memories, PUT for existing ones.
*/
export async function uploadMemory(
slug: string,
content: string,
existingState?: SyncedFileState,
): Promise<MemoryUploadResult> {
try {
const auth = await getAuthHeaders()
if (auth.error) {
return {
success: false,
slug,
error: auth.error,
errorType: 'auth',
}
}
const { frontmatter } = parseFrontmatter(content)
const name = (frontmatter.name as string) || slug
const description = (frontmatter.description as string) || ''
const type = (frontmatter.type as string) || 'user'
const projectPath = getProjectRoot() || process.cwd()
const workDir = process.cwd()
if (existingState?.memoryId) {
// Existing memory → PUT to update
const body: UpdateMemoryRequest = {
name,
description,
content,
bumpVersion: true,
}
const url = getMemoryAPIEndpoint(`/${existingState.memoryId}`)
const response = await fetch(url, {
method: 'PUT',
headers: auth.headers,
body: JSON.stringify(body),
signal: AbortSignal.timeout(API_TIMEOUT_MS),
})
if (!response.ok) {
return {
success: false,
slug,
error: `PUT failed: ${response.status} ${response.statusText}`,
errorType: response.status >= 500 ? 'network' : 'unknown',
httpStatus: response.status,
}
}
const data = (await response.json()) as MemoryResponse
return {
success: true,
slug,
memoryId: data.id,
version: data.currentVersion,
}
}
// New memory → POST to create
const body: CreateMemoryRequest = {
name,
slug,
projectPath,
workDir,
type,
description,
content,
}
const url = getMemoryAPIEndpoint('')
const response = await fetch(url, {
method: 'POST',
headers: auth.headers,
body: JSON.stringify(body),
signal: AbortSignal.timeout(API_TIMEOUT_MS),
})
if (!response.ok) {
return {
success: false,
slug,
error: `POST failed: ${response.status} ${response.statusText}`,
errorType: response.status >= 500 ? 'network' : 'unknown',
httpStatus: response.status,
}
}
const data = (await response.json()) as MemoryResponse
return {
success: true,
slug,
memoryId: data.id,
version: data.currentVersion,
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err)
const isTimeout =
message.includes('timeout') || message.includes('aborted')
return {
success: false,
slug,
error: message,
errorType: isTimeout ? 'timeout' : 'network',
}
}
}
// ─── Full scan & sync ──────────────────────────────────────────
/**
* Scan all .md files in the auto-memory directory and sync them to the cloud.
* Skips MEMORY.md and files in logs/ directory.
* Returns the set of slugs that were successfully synced.
*/
export async function scanAndSync(
existingState: CloudSyncState,
): Promise<{ results: MemoryUploadResult[]; newState: CloudSyncState }> {
const memoryDir = getAutoMemPath()
const newState: CloudSyncState = { items: { ...existingState.items } }
const results: MemoryUploadResult[] = []
let entries: string[]
try {
entries = await readdir(memoryDir, { recursive: true })
} catch {
// Memory dir doesn't exist yet — nothing to sync
return { results, newState }
}
const mdFiles = entries.filter(
f =>
f.endsWith('.md') &&
basename(f) !== 'MEMORY.md' &&
!f.startsWith('logs/') &&
!f.startsWith('logs\\'),
)
// Process files sequentially to avoid overwhelming the API
for (const relativePath of mdFiles) {
const slug = basename(relativePath, '.md')
const filePath = join(memoryDir, relativePath)
try {
const raw = await readFile(filePath, 'utf-8')
const md5 = computeMD5(raw)
const existing = newState.items[slug]
// Skip if content hasn't changed
if (existing && existing.contentMD5 === md5) {
continue
}
const result = await uploadMemory(slug, raw, existing)
if (result.success && result.memoryId) {
newState.items[slug] = {
memoryId: result.memoryId,
contentMD5: md5,
version: result.version ?? (existing?.version ?? 0) + 1,
}
}
results.push(result)
} catch (err) {
results.push({
success: false,
slug,
error: err instanceof Error ? err.message : String(err),
errorType: 'unknown',
})
}
}
return { results, newState }
}

View File

@ -0,0 +1,77 @@
/**
* Memory Cloud Sync Types
*
* Type definitions for the memory cloud sync API, which uploads
* auto-memory files to the costrict-web /api/memories endpoint.
*/
/**
* POST /api/memories request body
*/
export type CreateMemoryRequest = {
name: string
slug: string
projectPath: string
workDir: string
type: string
description: string
content: string
}
/**
* PUT /api/memories/:id request body
*/
export type UpdateMemoryRequest = {
name?: string
description?: string
content?: string
bumpVersion: boolean
}
/**
* Response from creating or fetching a memory
*/
export type MemoryResponse = {
id: string
userId: string
projectPath: string
slug: string
name: string
type: string
description?: string
currentVersion: number
createdAt: string
updatedAt: string
}
/**
* Per-file sync state stored in .cloud_sync_state.json
*/
export type SyncedFileState = {
/** Server-side memory ID */
memoryId: string
/** MD5 hash of the file content at last sync */
contentMD5: string
/** Server-side version number */
version: number
}
/**
* Full sync state file content
*/
export type CloudSyncState = {
items: Record<string, SyncedFileState>
}
/**
* Result of uploading a single memory file
*/
export type MemoryUploadResult = {
success: boolean
slug: string
memoryId?: string
version?: number
error?: string
errorType?: 'auth' | 'network' | 'timeout' | 'unknown'
httpStatus?: number
}

View File

@ -0,0 +1,313 @@
/**
* Memory Cloud Sync File Watcher
*
* On startup: scans all existing .md files and syncs them to the cloud.
* After startup: watches the auto-memory directory for changes and triggers
* a debounced push to the server when files are modified.
*/
import { type FSWatcher, watch } from 'fs'
import { mkdir, readFile, unlink, writeFile } from 'fs/promises'
import { join } from 'path'
import { getAutoMemPath, isAutoMemoryEnabled } from '../../../memdir/paths.js'
import { loadCoStrictCredentials } from '../../../costrict/provider/credentials.js'
import { registerCleanup } from '../../../utils/cleanupRegistry.js'
import { logForDebugging } from '../../../utils/debug.js'
import { isEnvTruthy } from '../../../utils/envUtils.js'
import { errorMessage } from '../../../utils/errors.js'
import { isProcessRunning } from '../../../utils/genericProcessUtils.js'
import {
loadSyncState,
saveSyncState,
scanAndSync,
} from './index.js'
const DEBOUNCE_MS = 10_000 // Wait 10s after last change before pushing
const LOCK_FILE = '.cloud_sync.lock'
// PID recycling guard: if the lock is older than this, reclaim even if PID is alive
const LOCK_STALE_MS = 60 * 60 * 1000 // 1 hour
// ─── Watcher state ─────────────────────────────────────────────
let watcher: FSWatcher | null = null
let debounceTimer: ReturnType<typeof setTimeout> | null = null
let pushInProgress = false
let hasPendingChanges = false
let watcherStarted = false
let lockAcquired = false
// ─── Process lock ─────────────────────────────────────────────
function lockPath(): string {
return join(getAutoMemPath(), LOCK_FILE)
}
/**
* Try to acquire the cloud sync lock. Only one process per project should
* run the watcher others skip to avoid duplicating uploads and corrupting
* the shared .cloud_sync_state.json.
*/
async function tryAcquireLock(): Promise<boolean> {
const path = lockPath()
try {
const raw = await readFile(path, 'utf8')
const holderPid = parseInt(raw.trim(), 10)
if (Number.isFinite(holderPid) && isProcessRunning(holderPid)) {
const stat = await import('fs/promises').then(m => m.stat(path))
const age = Date.now() - stat.mtimeMs
if (age < LOCK_STALE_MS) {
logForDebugging(
`memory-cloud-sync: lock held by live PID ${holderPid}, skipping`,
{ level: 'debug' },
)
return false
}
// Lock is stale — reclaim below
}
} catch {
// ENOENT — no prior lock, proceed
}
await mkdir(getAutoMemPath(), { recursive: true })
await writeFile(path, String(process.pid))
// Re-read to verify we won the race
const verify = await readFile(path, 'utf8')
if (parseInt(verify.trim(), 10) !== process.pid) {
logForDebugging(
'memory-cloud-sync: lost lock race, skipping',
{ level: 'debug' },
)
return false
}
return true
}
async function releaseLock(): Promise<void> {
if (!lockAcquired) return
try {
const path = lockPath()
const raw = await readFile(path, 'utf8')
if (parseInt(raw.trim(), 10) === process.pid) {
await unlink(path)
}
} catch {
// Best-effort
}
lockAcquired = false
}
/**
* Execute the push: re-scan all files and upload changed ones.
*/
async function executePush(): Promise<void> {
pushInProgress = true
try {
const state = await loadSyncState()
const { results, newState } = await scanAndSync(state)
await saveSyncState(newState)
const succeeded = results.filter(r => r.success).length
const failed = results.filter(r => !r.success).length
if (succeeded > 0 || failed > 0) {
logForDebugging(
`memory-cloud-sync: pushed ${succeeded} files${
failed > 0 ? `, ${failed} failed` : ''
}`,
{ level: 'info' },
)
}
if (failed === 0) {
hasPendingChanges = false
}
// Log individual failures for debugging
for (const r of results) {
if (!r.success) {
logForDebugging(
`memory-cloud-sync: upload failed for "${r.slug}": ${r.error}`,
{ level: 'warn' },
)
}
}
} catch (e) {
logForDebugging(`memory-cloud-sync: push error: ${errorMessage(e)}`, {
level: 'warn',
})
} finally {
pushInProgress = false
}
}
/**
* Debounced push: waits for writes to settle, then pushes once.
*/
function schedulePush(): void {
hasPendingChanges = true
if (debounceTimer) {
clearTimeout(debounceTimer)
}
debounceTimer = setTimeout(() => {
debounceTimer = null
if (pushInProgress) {
// A push is already running — mark pending and let it pick up
// the new changes on its next iteration
hasPendingChanges = true
return
}
void executePush()
}, DEBOUNCE_MS)
}
/**
* Start watching the auto-memory directory for changes.
*/
async function startFileWatcher(memoryDir: string): Promise<void> {
if (watcherStarted) {
return
}
watcherStarted = true
try {
await mkdir(memoryDir, { recursive: true })
watcher = watch(
memoryDir,
{ persistent: true, recursive: true },
(_eventType, _filename) => {
schedulePush()
},
)
watcher.on('error', err => {
logForDebugging(
`memory-cloud-sync: fs.watch error: ${errorMessage(err)}`,
{ level: 'warn' },
)
})
logForDebugging(`memory-cloud-sync: watching ${memoryDir}`, {
level: 'debug',
})
} catch (err) {
logForDebugging(
`memory-cloud-sync: failed to watch ${memoryDir}: ${errorMessage(err)}`,
{ level: 'warn' },
)
}
registerCleanup(async () => stopMemoryCloudWatcher())
}
/**
* Start the memory cloud sync system.
*
* Enabled by default. Set DISABLE_MEMORY_CLOUD_SYNC=1 to opt out.
*
* Returns early if:
* - DISABLE_MEMORY_CLOUD_SYNC env var is truthy
* - auto memory is disabled
* - user has no CoStrict credentials
*
* On start: performs a full scan of existing memory files and uploads
* any that are new or changed. Then starts the file watcher.
*/
export async function startMemoryCloudWatcher(): Promise<void> {
if (isEnvTruthy(process.env.DISABLE_MEMORY_CLOUD_SYNC)) {
logForDebugging(
'memory-cloud-sync: disabled via DISABLE_MEMORY_CLOUD_SYNC',
{ level: 'debug' },
)
return
}
if (!isAutoMemoryEnabled()) {
logForDebugging(
'memory-cloud-sync: auto memory disabled, skipping',
{ level: 'debug' },
)
return
}
const creds = await loadCoStrictCredentials()
if (!creds?.access_token) {
logForDebugging(
'memory-cloud-sync: no CoStrict credentials, skipping',
{ level: 'debug' },
)
return
}
const memoryDir = getAutoMemPath()
// Acquire process lock — only one watcher per project
const gotLock = await tryAcquireLock()
if (!gotLock) {
return
}
lockAcquired = true
logForDebugging(
`memory-cloud-sync: starting initial scan of ${memoryDir}`,
{ level: 'info' },
)
// ── Initial full scan ──────────────────────────────────────────
try {
const state = await loadSyncState()
const { results, newState } = await scanAndSync(state)
await saveSyncState(newState)
const succeeded = results.filter(r => r.success).length
const failed = results.filter(r => !r.success).length
logForDebugging(
`memory-cloud-sync: initial scan complete — ${succeeded} uploaded${
failed > 0 ? `, ${failed} failed` : ''
}`,
{ level: 'info' },
)
} catch (err) {
logForDebugging(
`memory-cloud-sync: initial scan failed: ${errorMessage(err)}`,
{ level: 'warn' },
)
}
// ── Start file watcher ─────────────────────────────────────────
await startFileWatcher(memoryDir)
}
/**
* Stop the file watcher and flush pending changes.
*/
export async function stopMemoryCloudWatcher(): Promise<void> {
if (debounceTimer) {
clearTimeout(debounceTimer)
debounceTimer = null
}
if (watcher) {
watcher.close()
watcher = null
}
// Flush pending changes
if (hasPendingChanges) {
try {
await executePush()
} catch {
// Best-effort during shutdown
}
}
await releaseLock()
}
/**
* Test-only: reset module state for test isolation.
*/
export function _resetWatcherStateForTesting(): void {
watcher = null
debounceTimer = null
pushInProgress = false
hasPendingChanges = false
watcherStarted = false
}

View File

@ -363,6 +363,9 @@ export async function setup(
m.startTeamMemoryWatcher(),
) // Start team memory sync watcher
}
void import('./costrict/services/memoryCloudSync/watcher.js').then(m =>
m.startMemoryCloudWatcher(),
) // Start memory cloud sync watcher (disabled via DISABLE_MEMORY_CLOUD_SYNC=1)
}
initSinks() // Attach error log + analytics sinks and drain queued events