Initial incoming post -> channel~

Also adds channel "username" like access via url/+channel-slug (a new, 
required, and currently unsettable, channel option)

Completely untested, and known unfinshed in a few places, good luck 
future me!

packages/backend/migration/1653969505036-channelBoxes.js

@@ -0,0 +1,45 @@
+import { toPuny } from '../built/misc/convert-host.js';
+
+export class channelBoxes1653969505036 {
+    name = 'channelBoxes1653969505036'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "channel" ADD "slug" character varying(512)`);
+        await queryRunner.query(`COMMENT ON COLUMN "channel"."slug" IS 'URL safe channel name.'`);
+        const channels = await queryRunner.query(`SELECT id, name FROM "channel"`);
+        for (let i = 0; i < channels.length; i++) {
+          const channelId = channels[i].id;
+          const channelName = channels[i].name;
+          const slug = toPuny(channelName.replace(' ', '-'));
+
+          await queryRunner.query(`UPDATE "channel" SET "slug"='${slug}' WHERE "id" = '${channelId}'`);
+        }
+        await queryRunner.query(`ALTER TABLE "channel" ALTER COLUMN "slug" SET NOT NULL`);
+
+        await queryRunner.query(`ALTER TABLE "channel" ADD "host" character varying(128)`);
+        await queryRunner.query(`COMMENT ON COLUMN "channel"."host" IS 'The host of the Channel. It will be null if the origin of the channel is local.'`);
+        await queryRunner.query(`ALTER TABLE "channel" ADD "inbox" character varying(512)`);
+        await queryRunner.query(`COMMENT ON COLUMN "channel"."inbox" IS 'The inbox URL of the Channel. It will be null if the origin of the channel is local.'`);
+        await queryRunner.query(`ALTER TABLE "channel" ADD "sharedInbox" character varying(512)`);
+        await queryRunner.query(`COMMENT ON COLUMN "channel"."sharedInbox" IS 'The sharedInbox URL of the Channel. It will be null if the origin of the channel is local.'`);
+        await queryRunner.query(`ALTER TABLE "channel" ADD "uri" character varying(512)`);
+        await queryRunner.query(`COMMENT ON COLUMN "channel"."uri" IS 'The URI of the Channel. It will be null if the origin of the channel is local.'`);
+        await queryRunner.query(`ALTER TABLE "channel" ADD "followersUri" character varying(512)`);
+        await queryRunner.query(`COMMENT ON COLUMN "channel"."followersUri" IS 'The URI of the channel Follower Collection. It will be null if the origin of the channel is local.'`);
+
+        await queryRunner.query(`CREATE INDEX "IDX_8479b8802f0cd5d60c61a5ad20" ON "channel" ("slug") `);
+        await queryRunner.query(`CREATE INDEX "IDX_ddc763034a97bf9035c8a6bd12" ON "channel" ("host") `);
+        await queryRunner.query(`CREATE INDEX "IDX_bd0e56edb9ccb54f92b3693d14" ON "channel" ("uri") `);
+        await queryRunner.query(`CREATE UNIQUE INDEX "IDX_4bfbb3a1ec83af58d47b298942" ON "channel" ("slug", "host") `);
+
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "channel" DROP COLUMN "followersUri"`);
+        await queryRunner.query(`ALTER TABLE "channel" DROP COLUMN "uri"`);
+        await queryRunner.query(`ALTER TABLE "channel" DROP COLUMN "sharedInbox"`);
+        await queryRunner.query(`ALTER TABLE "channel" DROP COLUMN "inbox"`);
+        await queryRunner.query(`ALTER TABLE "channel" DROP COLUMN "host"`);
+        await queryRunner.query(`ALTER TABLE "channel" DROP COLUMN "slug"`);
+    }
+}

packages/backend/src/misc/keypair-store.ts

@@ -9,7 +9,7 @@ const userCache = new Cache<UserKeypair>(Infinity);
 const channelCache = new Cache<ChannelKeypair>(Infinity);
 
 export async function getUserKeypair(userId: User['id']): Promise<UserKeypair> {
-	return await cache.fetch(userId, () => UserKeypairs.findOneByOrFail({ userId }));
+	return await userCache.fetch(userId, () => UserKeypairs.findOneByOrFail({ userId }));
 }
 
 export async function getChannelKeypair(channelId: Channel['id']): Promise<ChannelKeypair> {

packages/backend/src/models/entities/channel.ts

@@ -4,6 +4,7 @@ import { id } from '../id.js';
 import { DriveFile } from './drive-file.js';
 
 @Entity()
+@Index(['slug', 'host'], { unique: true })
 export class Channel {
 	@PrimaryColumn(id())
 	public id: string;
@@ -40,6 +41,13 @@ export class Channel {
 	})
 	public name: string;
 
+	@Index()
+	@Column('varchar', {
+		length: 512,
+		comment: 'URL safe channel name.',
+	})
+	public slug: string;
+
 	@Column('varchar', {
 		length: 2048, nullable: true,
 		comment: 'The description of the Channel.',
@@ -83,4 +91,36 @@ export class Channel {
 		comment: 'The count of users.',
 	})
 	public usersCount: number;
+
+	@Index()
+	@Column('varchar', {
+		length: 128, nullable: true,
+		comment: 'The host of the Channel. It will be null if the origin of the channel is local.',
+	})
+	public host: string | null;
+
+	@Column('varchar', {
+		length: 512, nullable: true,
+		comment: 'The inbox URL of the Channel. It will be null if the origin of the channel is local.',
+	})
+	public inbox: string | null;
+
+	@Column('varchar', {
+		length: 512, nullable: true,
+		comment: 'The sharedInbox URL of the Channel. It will be null if the origin of the channel is local.',
+	})
+	public sharedInbox: string | null;
+
+	@Index()
+	@Column('varchar', {
+		length: 512, nullable: true,
+		comment: 'The URI of the Channel. It will be null if the origin of the channel is local.',
+	})
+	public uri: string | null;
+
+	@Column('varchar', {
+		length: 512, nullable: true,
+		comment: 'The URI of the channel Follower Collection. It will be null if the origin of the channel is local.',
+	})
+	public followersUri: string | null;
 }

packages/backend/src/remote/activitypub/models/group.ts

@@ -0,0 +1,250 @@
+import { URL } from 'node:url';
+import promiseLimit from 'promise-limit';
+
+import $, { Context } from 'cafy';
+import config from '@/config/index.js';
+import Resolver from '../resolver.js';
+import { resolveImage } from './image.js';
+import { isCollectionOrOrderedCollection, isCollection, IActor, getApId, getOneApHrefNullable, IObject, isPropertyValue, IApPropertyValue, getApType, isActor } from '../type.js';
+import { fromHtml } from '../../../mfm/from-html.js';
+import { htmlToMfm } from '../misc/html-to-mfm.js';
+import { resolveNote, extractEmojis } from './note.js';
+import { registerOrFetchInstanceDoc } from '@/services/register-or-fetch-instance-doc.js';
+import { extractApHashtags } from './tag.js';
+import { apLogger } from '../logger.js';
+import { Note } from '@/models/entities/note.js';
+import { updateUsertags } from '@/services/update-hashtag.js';
+import { Users, Instances, DriveFiles, Followings, UserProfiles, UserPublickeys } from '@/models/index.js';
+import { User, IRemoteUser, CacheableUser } from '@/models/entities/user.js';
+import { Emoji } from '@/models/entities/emoji.js';
+import { UserNotePining } from '@/models/entities/user-note-pining.js';
+import { genId } from '@/misc/gen-id.js';
+import { instanceChart, usersChart } from '@/services/chart/index.js';
+import { UserPublickey } from '@/models/entities/user-publickey.js';
+import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
+import { toPuny } from '@/misc/convert-host.js';
+import { UserProfile } from '@/models/entities/user-profile.js';
+import { toArray } from '@/prelude/array.js';
+import { fetchInstanceMetadata } from '@/services/fetch-instance-metadata.js';
+import { normalizeForSearch } from '@/misc/normalize-for-search.js';
+import { truncate } from '@/misc/truncate.js';
+import { StatusError } from '@/misc/fetch.js';
+import { uriPersonCache } from '@/services/user-cache.js';
+import { publishInternalEvent } from '@/services/stream.js';
+import { db } from '@/db/postgre.js';
+
+const logger = apLogger;
+
+const nameLength = 128;
+const summaryLength = 2048;
+
+/**
+ * Validate and convert to actor object
+ * @param x Fetched object
+ * @param uri Fetch target URI
+ */
+function validateActor(x: IObject, uri: string): IActor {
+	return x;
+}
+
+export async function fetchGroup(uri: string, resolver?: Resolver): Promise<CacheableGroup | null> {
+	if (typeof uri !== 'string') throw new Error('uri is not string');
+
+	const exist = await Channel.findOneBy({ uri });
+
+	if (exist) {
+		return exist;
+	}
+	return null;
+}
+
+export async function createGroup(uri: string, resolver?: Resolver): Promise<Channel> {
+	if (typeof uri !== 'string') throw new Error('uri is not string');
+
+	if (uri.startsWith(config.url)) {
+		throw new StatusError('cannot resolve local channel', 400, 'cannot resolve local channel');
+	}
+
+	if (resolver == null) resolver = new Resolver();
+
+	const object = await resolver.resolve(uri) as any;
+
+	const group = validateActor(object, uri);
+
+	logger.info(`Creating the Group: ${group.id}`);
+
+	const host = toPuny(new URL(group.id).hostname);
+
+	const tags = extractApHashtags(group.tag).map(tag => normalizeForSearch(tag)).splice(0, 32);
+
+	// Create channel
+	let channel: IRemoteChannel;
+	try {
+		// Start transaction
+		await db.transaction(async transactionalEntityManager => {
+			channel = await transactionalEntityManager.save(new Channel({
+				id: genId(),
+				name: truncate(person.name, nameLength),
+				host,
+				inbox: person.inbox,
+				sharedInbox: person.sharedInbox || (person.endpoints ? person.endpoints.sharedInbox : undefined),
+				followersUri: person.followers ? getApId(person.followers) : undefined,
+				uri: person.id,
+				tags,
+			})) as IRemoteChannel;
+
+			// if (person.publicKey) {
+			// 	await transactionalEntityManager.save(new UserPublickey({
+			// 		userId: user.id,
+			// 		keyId: person.publicKey.id,
+			// 		keyPem: person.publicKey.publicKeyPem,
+			// 	}));
+			// }
+		});
+	} catch (e) {
+		// duplicate key error
+		if (isDuplicateKeyValueError(e)) {
+			throw new Error('already registered');
+		} else {
+			logger.error(e instanceof Error ? e : new Error(e as string));
+			throw e;
+		}
+	}
+
+	// Register host
+	registerOrFetchInstanceDoc(host).then(i => {
+		fetchInstanceMetadata(i);
+	});
+
+	//#region アバターとヘッダー画像をフェッチ
+	let banner;
+	if (banner) {
+		banner = await resolveImage(channel!, group.image).catch(() => null)
+	}
+
+	const bannerId = banner ? banner.id : null;
+
+	await Channel.update(channel!.id, {
+		bannerId,
+	});
+
+	channel!.bannerId = bannerId;
+	//#endregion
+
+	//#region カスタム絵文字取得
+	const emojis = await extractEmojis(group.tag || [], host).catch(e => {
+		logger.info(`extractEmojis: ${e}`);
+		return [] as Emoji[];
+	});
+
+	const emojiNames = emojis.map(emoji => emoji.name);
+
+	await Channel.update(channel!.id, {
+		emojis: emojiNames,
+	});
+	//#endregion
+
+	return channel!;
+}
+
+/**
+ * Personの情報を更新します。
+ * Misskeyに対象のPersonが登録されていなければ無視します。
+ * @param uri URI of Person
+ * @param resolver Resolver
+ * @param hint Hint of Person object (この値が正当なPersonの場合、Remote resolveをせずに更新に利用します)
+ */
+export async function updateGroup(uri: string, resolver?: Resolver | null, hint?: Record<string, unknown>): Promise<void> {
+	if (typeof uri !== 'string') throw new Error('uri is not string');
+
+	// URIがこのサーバーを指しているならスキップ
+	if (uri.startsWith(config.url + '/')) {
+		return;
+	}
+
+	//#region このサーバーに既に登録されているか
+	const exist = await Channel.findOneBy({ uri }) as IRemoteChannel;
+
+	if (exist == null) {
+		return;
+	}
+	//#endregion
+
+	if (resolver == null) resolver = new Resolver();
+
+	const object = hint || await resolver.resolve(uri) as any;
+
+	const group = validateActor(object, uri);
+
+	logger.info(`Updating the Group: ${group.id}`);
+
+	// アバターとヘッダー画像をフェッチ
+	let banner;
+	if (banner) {
+		banner = await resolveImage(channel!, group.image).catch(() => null)
+	}
+
+	// カスタム絵文字取得
+	const emojis = await extractEmojis(group.tag || [], exist.host).catch(e => {
+		logger.info(`extractEmojis: ${e}`);
+		return [] as Emoji[];
+	});
+
+	const emojiNames = emojis.map(emoji => emoji.name);
+
+	const tags = extractApHashtags(group.tag).map(tag => normalizeForSearch(tag)).splice(0, 32);
+
+	const updates = {
+		inbox: group.inbox,
+		sharedInbox: group.sharedInbox || (group.endpoints ? group.endpoints.sharedInbox : undefined),
+		followersUri: group.followers ? getApId(group.followers) : undefined,
+		emojis: emojiNames,
+		name: truncate(group.name, nameLength),
+		tags,
+	} as Partial<User>;
+
+	if (banner) {
+		updates.bannerId = banner.id;
+	}
+
+	// Update user
+	await Channel.update(exist.id, updates);
+
+	// if (person.publicKey) {
+	// 	await UserPublickeys.update({ userId: exist.id }, {
+	// 		keyId: person.publicKey.id,
+	// 		keyPem: person.publicKey.publicKeyPem,
+	// 	});
+	// }
+
+	// TODO: investigate if this should happen for channels aswell
+	// publishInternalEvent('remoteUserUpdated', { id: exist.id });
+
+	await Followings.update({
+		followerId: exist.id,
+	}, {
+		followerSharedInbox: group.sharedInbox || (group.endpoints ? group.endpoints.sharedInbox : undefined),
+	});
+}
+
+/**
+ * Personを解決します。
+ *
+ * Misskeyに対象のPersonが登録されていればそれを返し、そうでなければ
+ * リモートサーバーからフェッチしてMisskeyに登録しそれを返します。
+ */
+export async function resolveGroup(uri: string, resolver?: Resolver): Promise<CacheableGroup> {
+	if (typeof uri !== 'string') throw new Error('uri is not string');
+
+	//#region このサーバーに既に登録されていたらそれを返す
+	const exist = await fetchGroup(uri);
+
+	if (exist) {
+		return exist;
+	}
+	//#endregion
+
+	// リモートサーバーからフェッチしてきて登録
+	if (resolver == null) resolver = new Resolver();
+	return await createGroup(uri, resolver);
+}

packages/backend/src/remote/activitypub/models/note.ts

@@ -5,6 +5,7 @@ import Resolver from '../resolver.js';
 import post from '@/services/note/create.js';
 import { resolvePerson, updatePerson } from './person.js';
 import { resolveImage } from './image.js';
+import { resolveGroup, updateGroup } from './group.js';
 import { CacheableRemoteUser, IRemoteUser } from '@/models/entities/user.js';
 import { htmlToMfm } from '../misc/html-to-mfm.js';
 import { extractApHashtags } from './tag.js';
@@ -226,6 +227,11 @@ export async function createNote(value: string | IObject, resolver?: Resolver, s
 		return [] as Emoji[];
 	});
 
+	const channel = await extractChannel(note.tag || [], actor.host).catch(e => {
+		logger.info(`extractChannel: ${e}`);
+		return null;
+	});
+
 	const apEmojis = emojis.map(emoji => emoji.name);
 
 	const poll = await extractPollFromQuestion(note, resolver).catch(() => undefined);
@@ -252,6 +258,7 @@ export async function createNote(value: string | IObject, resolver?: Resolver, s
 		apHashtags,
 		apEmojis,
 		poll,
+		channel,
 		uri: note.id,
 		url: getOneApHrefNullable(note.url),
 	}, silent);
@@ -350,3 +357,14 @@ export async function extractEmojis(tags: IObject | IObject[], host: string): Pr
 		} as Partial<Emoji>).then(x => Emojis.findOneByOrFail(x.identifiers[0]));
 	}));
 }
+
+
+export async function extractChannel(tags: IObject | IObject[], host:string): Promise<Channel | null> {
+	const apGroup = toArray(tags).find(isGroup);
+
+	if (!apGroup) {
+		return null;
+	}
+
+	return await resolveGroup(getOneApId(apGroup), resolver) as CacheableRemoteGroup;
+}

packages/backend/src/remote/activitypub/renderer/group.ts

@@ -38,7 +38,7 @@ export async function renderGroup(channel: Channel) {
 		following: `${id}/following`, // this is required by spec. cant be empty. sadge
 		sharedInbox: `${config.url}/inbox`,
 		endpoints: { sharedInbox: `${config.url}/inbox` },
-		url: id,
+		url: `${config.url}/+${channel.slug}`,
 		name: channel.name,
 		summary: channel.description ? toHtml(mfm.parse(channel.description)) : null,
 		icon: null,

packages/backend/src/remote/activitypub/renderer/note.ts

@@ -2,6 +2,7 @@ import renderDocument from './document.js';
 import renderHashtag from './hashtag.js';
 import renderMention from './mention.js';
 import renderEmoji from './emoji.js';
+import renderGroup from './group.js';
 import config from '@/config/index.js';
 import toHtml from '../misc/get-note-html.js';
 import { Note, IMentionedRemoteUsers } from '@/models/entities/note.js';
@@ -53,6 +54,12 @@ export default async function renderNote(note: Note, dive = true, isTalk = false
 		}
 	}
 
+	let channel: Channel | null;
+
+	if (note.channelId) {
+		channel = await Channel.findOneBy({ id: note.channelId });
+	}
+
 	const attributedTo = `${config.url}/users/${note.userId}`;
 
 	const mentions = (JSON.parse(note.mentionedRemoteUsers) as IMentionedRemoteUsers).map(x => x.uri);
@@ -60,9 +67,10 @@ export default async function renderNote(note: Note, dive = true, isTalk = false
 	let to: string[] = [];
 	let cc: string[] = [];
 
-	if (note.channelId) {
+	if (channel) {
-		const channel = `${config.url}/channels/${note.channelId}`;
+		const channelUrl = `${config.url}/channels/${channel.id}`;
-		to = [channel];
+		// to = [`${channelUrl}/followers`];
+		to = [channelUrl];
 		cc = [`${attributedTo}/followers`, 'https://www.w3.org/ns/activitystreams#Public'].concat(mentions);
 	} else {
 		if (note.visibility === 'public') {
@@ -112,11 +120,15 @@ export default async function renderNote(note: Note, dive = true, isTalk = false
 	const emojis = await getEmojis(note.emojis);
 	const apemojis = emojis.map(emoji => renderEmoji(emoji));
 
-	const tag = [
+	let tag = [
 		...hashtagTags,
 		...mentionTags,
 		...apemojis,
 	];
+	if (channel) {
+		const apGroup = await renderGroup(channel);
+		tag.push(apGroup);
+	}
 
 	const asPoll = poll ? {
 		type: 'Question',

packages/backend/src/remote/activitypub/type.ts

@@ -216,6 +216,14 @@ export interface IApEmoji extends IObject {
 export const isEmoji = (object: IObject): object is IApEmoji =>
 	getApType(object) === 'Emoji' && !Array.isArray(object.icon) && object.icon.url != null;
 
+export interface IApGroup extends IObject {
+	type: 'Group';
+	id: String;
+}
+
+export const isGroup = (object: IObject): object is IApGroup =>
+	getApType(object) === 'Group' && typeof object.id === 'string';
+
 export interface ICreate extends IActivity {
 	type: 'Create';
 }

packages/backend/src/server/activitypub.ts

@@ -226,21 +226,38 @@ router.get('/likes/:like', async ctx => {
 });
 
 // channel
-router.get('/channels/:channelId', async (ctx, next) => {
+async function channelInfo(ctx: Router.RouterContext, channel: Channel | null) {
-	if (!isActivityPubReq(ctx)) return await next();
-
-	const channel = await Channels.findOneBy({
-		id: ctx.params.channelId,
-	});
-
 	if (channel == null) {
 		ctx.status = 404;
 		return;
 	}
 
-	ctx.body = renderActivity(await renderGroup(channel));
+	ctx.body = renderActivity(await renderGroup(channel as ILocalChannel));
 	ctx.set('Cache-Control', 'public, max-age=180');
 	setResponseType(ctx);
+}
+
+router.get('/channels/:channelId', async (ctx, next) => {
+	if (!isActivityPubReq(ctx)) return await next();
+
+	const channel = await Channels.findOneBy({
+		id: ctx.params.channelId,
+		host: IsNull(),
+	});
+
+	await channelInfo(ctx, channel);
 });
 
+router.get('/\\+:channelSlug', async (ctx, next) => {
+	if (!isActivityPubReq(ctx)) return await next();
+
+	const channel = await Channels.findOneBy({
+		slug: ctx.params.channelSlug.toLowerCase(),
+		host: IsNull(),
+	});
+
+	await channelInfo(ctx, channel);
+});
+//#endregion
+
 export default router;

packages/backend/src/server/api/endpoints/channels/create.ts

@@ -54,17 +54,18 @@ export default define(meta, paramDef, async (ps, user) => {
 		}
 	}
 
-	const keyPair = genRsaKeyPair(4096);
+	const keyPair = await genRsaKeyPair(4096);
 
+	let channel;
 	await db.transaction(async transactionalEntityManager => {
-		const channel = await transactionalEntityManager.insert(Channel, {
+		channel = await transactionalEntityManager.insert(Channel, {
 			id: genId(),
 			createdAt: new Date(),
 			userId: user.id,
 			name: ps.name,
 			description: ps.description || null,
 			bannerId: banner ? banner.id : null,
-		}).then(x => Channels.findOneByOrFail(x.identifiers[0]));
+		}).then(x => transactionalEntityManager.findOneByOrFail(Channel, x.identifiers[0]));
 
 		await transactionalEntityManager.insert(ChannelKeypair, {
 			publicKey: keyPair.publicKey,