pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add workbench kafka migration support

Browse Source
This commit is contained in:
richjohnrambo
2026-09-13 22:17:02 +08:00
parent c46cb4b382
commit 67e8c2c6f6
211 changed files with 7391 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/db/mysql/adapter.go
+13 -423
View File
@@ -269,12 +269,6 @@ func (a *adapter) Open(jsonconfig json.RawMessage) error {
// Actually opening the network connection.
err = a.db.Ping()
databaseMissing := isMissingDb(err)
if databaseMissing {
// Ignore missing database here. If we are initializing the database
// missing DB is OK.
err = nil
}
if err == nil {
if config.SqlTimeout > 0 {
a.sqlTimeout = time.Duration(config.SqlTimeout) * time.Second
@@ -611,435 +605,31 @@ func createTenantRegistry(tx *sql.Tx, registryDB, businessDB string) error {
return err
}
// CreateDb initializes the storage.
// CreateDb is intentionally disabled for this deployment.
//
// Database creation, table creation, schema migration, indexes and seed data
// are handled by reviewed SQL files and the release process. The service and
// local database tools must not create or mutate schema implicitly.
func (a *adapter) CreateDb(reset bool) error {
var err error
var tx *sql.Tx
// Can't use an existing connection because it's configured with a database name which may not exist.
// Don't care if it does not close cleanly.
a.db.Close()
// This DSN has been parsed before and produced no error, not checking for errors here.
cfg, _ := ms.ParseDSN(a.dsn)
// Clear database name
cfg.DBName = ""
a.db, err = sqlx.Open(instrumentedMySQLDriverName, cfg.FormatDSN())
if err != nil {
return err
}
if tx, err = a.db.Begin(); err != nil {
return err
}
defer func() {
if err != nil {
// FIXME: This is useless: MySQL auto-commits on every CREATE TABLE.
// Maybe DROP DATABASE instead.
tx.Rollback()
}
}()
registryDB := a.tenantRegistryDB
if registryDB == "" {
registryDB = "hotgo"
}
businessDB := a.dbName
if businessDB == "" {
businessDB = defaultDatabase
}
if reset {
if _, err = tx.Exec("DROP DATABASE IF EXISTS " + quoteIdent(businessDB)); err != nil {
return err
}
if _, err = tx.Exec("CREATE DATABASE IF NOT EXISTS " + quoteIdent(registryDB) + " CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); err != nil {
return err
}
if _, err = tx.Exec("DROP TABLE IF EXISTS " + quoteIdent(registryDB) + ".im_tenant_config"); err != nil {
return err
}
if _, err = tx.Exec("DROP TABLE IF EXISTS " + quoteIdent(registryDB) + ".im_tenant"); err != nil {
return err
}
}
if err = createTenantRegistry(tx, registryDB, businessDB); err != nil {
return err
}
if _, err = tx.Exec("CREATE DATABASE IF NOT EXISTS " + quoteIdent(businessDB) + " CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci"); err != nil {
return err
}
if _, err = tx.Exec("USE " + quoteIdent(businessDB)); err != nil {
return err
}
if _, err = tx.Exec(
`CREATE TABLE im_users(
id BIGINT NOT NULL,
account_no INT UNSIGNED NOT NULL DEFAULT 0,
username VARCHAR(32) NOT NULL,
createdat DATETIME(3) NOT NULL,
updatedat DATETIME(3) NOT NULL,
state SMALLINT NOT NULL DEFAULT 0,
stateat DATETIME(3),
access JSON,
lastseen DATETIME,
useragent VARCHAR(255) DEFAULT '',
phone VARCHAR(32),
public JSON,
trusted JSON,
PRIMARY KEY(id),
UNIQUE INDEX uk_im_users_account_no(account_no),
UNIQUE INDEX users_username(username),
UNIQUE INDEX users_phone(phone),
INDEX users_state_stateat(state, stateat),
INDEX users_lastseen_updatedat(lastseen, updatedat)
)`); err != nil {
return err
}
if _, err = tx.Exec(
`CREATE TABLE im_user_security(
id BIGINT NOT NULL,
user_id BIGINT NOT NULL,
auth_version INT NOT NULL DEFAULT 1,
password_updated_at DATETIME(3),
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id),
UNIQUE INDEX uk_im_user_security_user(user_id)
)`); err != nil {
return err
}
if err = createIMSecurityTables(tx); err != nil {
return err
}
if err = createIMNotificationTables(tx); err != nil {
return err
}
if err = ensureIMRegistrationInviteUseTable(tx); err != nil {
return err
}
if a.registryDB == nil {
if err = ensureIMRegistrationInviteControlTables(tx); err != nil {
return err
}
}
// Indexed devices. Normalized into a separate table.
if _, err = tx.Exec(
`CREATE TABLE im_devices(
id INT NOT NULL AUTO_INCREMENT,
userid BIGINT NOT NULL,
hash CHAR(16) NOT NULL,
deviceid TEXT NOT NULL,
platform VARCHAR(32),
lastseen DATETIME NOT NULL,
lang VARCHAR(8),
PRIMARY KEY(id),
FOREIGN KEY(userid) REFERENCES im_users(id),
UNIQUE INDEX devices_hash(hash)
)`); err != nil {
return err
}
// Authentication records for the basic authentication scheme.
if _, err = tx.Exec(
`CREATE TABLE im_auth(
id INT NOT NULL AUTO_INCREMENT,
userid BIGINT NOT NULL,
authlvl INT NOT NULL,
secret VARCHAR(255) NOT NULL,
expires DATETIME,
PRIMARY KEY(id),
FOREIGN KEY(userid) REFERENCES im_users(id),
UNIQUE INDEX auth_userid(userid)
)`); err != nil {
return err
}
// Topics
if _, err = tx.Exec(
`CREATE TABLE im_topics(
id INT NOT NULL AUTO_INCREMENT,
createdat DATETIME(3) NOT NULL,
updatedat DATETIME(3) NOT NULL,
state SMALLINT NOT NULL DEFAULT 0,
stateat DATETIME(3),
touchedat DATETIME(3),
name CHAR(25) NOT NULL,
usebt TINYINT DEFAULT 0,
owner BIGINT NOT NULL DEFAULT 0,
access JSON,
seqid BIGINT NOT NULL DEFAULT 0,
delid INT DEFAULT 0,
subcnt INT DEFAULT 0,
public JSON,
trusted JSON,
tags JSON,
aux JSON,
PRIMARY KEY(id),
UNIQUE INDEX topics_name(name),
INDEX topics_owner(owner),
INDEX topics_state_stateat(state, stateat),
INDEX topics_name_state_seqid(name, state, seqid)
)`); err != nil {
return err
}
// Create system topic 'sys'.
if err = createSystemTopic(tx); err != nil {
return err
}
// Indexed topic tags.
if _, err = tx.Exec(
`CREATE TABLE im_topictags(
id INT NOT NULL AUTO_INCREMENT,
topic CHAR(25) NOT NULL,
tag VARCHAR(96) NOT NULL,
PRIMARY KEY(id),
FOREIGN KEY(topic) REFERENCES im_topics(name),
INDEX topictags_tag(tag),
UNIQUE INDEX topictags_topic_tag(topic, tag)
)`); err != nil {
return err
}
// Subscriptions
if _, err = tx.Exec(
`CREATE TABLE im_subscriptions(
id INT NOT NULL AUTO_INCREMENT,
createdat DATETIME(3) NOT NULL,
updatedat DATETIME(3) NOT NULL,
deletedat DATETIME(3),
userid BIGINT NOT NULL,
topic CHAR(25) NOT NULL,
delid INT DEFAULT 0,
recvseqid BIGINT DEFAULT 0,
readseqid BIGINT DEFAULT 0,
modewant CHAR(8),
modegiven CHAR(8),
private JSON,
PRIMARY KEY(id),
FOREIGN KEY(userid) REFERENCES im_users(id),
UNIQUE INDEX subscriptions_topic_userid(topic, userid),
INDEX subscriptions_topic(topic),
INDEX subscriptions_deletedat(deletedat),
INDEX subscriptions_userid_topic_deletedat(userid, topic, deletedat)
)`); err != nil {
return err
}
// Messages
if _, err = tx.Exec(
`CREATE TABLE im_messages(
id INT NOT NULL AUTO_INCREMENT,
createdat DATETIME(3) NOT NULL,
updatedat DATETIME(3) NOT NULL,
deletedat DATETIME(3),
delid INT DEFAULT 0,
seqid BIGINT NOT NULL,
topic CHAR(25) NOT NULL,` +
"`from` BIGINT NOT NULL," +
`client_msg_id VARCHAR(64),
version INT NOT NULL DEFAULT 1,
recalled_at DATETIME(3),
recalled_by BIGINT,
head JSON,
content JSON,
PRIMARY KEY(id),
FOREIGN KEY(topic) REFERENCES im_topics(name),
UNIQUE INDEX messages_sender_client_mid(` + "`from`" + `, client_msg_id),
UNIQUE INDEX messages_topic_seqid(topic, seqid)
)`); err != nil {
return err
}
if _, err = tx.Exec(
`CREATE TABLE im_message_revisions(
id BIGINT NOT NULL AUTO_INCREMENT,
createdat DATETIME(3) NOT NULL,
updatedat DATETIME(3) NOT NULL,
topic CHAR(25) NOT NULL,
seqid BIGINT NOT NULL,
version INT NOT NULL,
actor BIGINT NOT NULL,
head JSON,
content JSON,
reason VARCHAR(256) NOT NULL DEFAULT '',
PRIMARY KEY(id),
UNIQUE INDEX uk_im_message_revisions_version(topic, seqid, version)
)`); err != nil {
return err
}
if err = createIMRelationTables(tx); err != nil {
return err
}
if err = createIMCallTables(tx); err != nil {
return err
}
if err = createIMWorkbenchTables(tx); err != nil {
return err
}
if err = createIMOrganizationTables(tx); err != nil {
return err
}
if err = createIMAttendanceTables(tx); err != nil {
return err
}
if err = createIMMeetingTables(tx); err != nil {
return err
}
if err = createIMReportTables(tx); err != nil {
return err
}
if err = createIMMailboxTables(tx); err != nil {
return err
}
if err = createIMUserProductTables(tx); err != nil {
return err
}
if err = ensureIMGroupBusinessTables(tx); err != nil {
return err
}
if err = ensureIMHTTPInfrastructureTables(tx); err != nil {
return err
}
if err = ensureIMAdminGovernanceTables(tx); err != nil {
return err
}
if err = ensureIMFileReviewTables(tx); err != nil {
return err
}
// Deletion log
if _, err = tx.Exec(
`CREATE TABLE im_dellog(
id INT NOT NULL AUTO_INCREMENT,
topic CHAR(25) NOT NULL,
deletedfor BIGINT NOT NULL DEFAULT 0,
delid INT NOT NULL,
low INT NOT NULL,
hi INT NOT NULL,
PRIMARY KEY(id),
FOREIGN KEY(topic) REFERENCES im_topics(name),
INDEX dellog_topic_delid_deletedfor(topic,delid,deletedfor),
INDEX dellog_topic_deletedfor_low_hi(topic,deletedfor,low,hi),
INDEX dellog_deletedfor(deletedfor)
)`); err != nil {
return err
}
// User credentials
if _, err = tx.Exec(
`CREATE TABLE im_credentials(
id INT NOT NULL AUTO_INCREMENT,
createdat DATETIME(3) NOT NULL,
updatedat DATETIME(3) NOT NULL,
deletedat DATETIME(3),
method VARCHAR(16) NOT NULL,
value VARCHAR(128) NOT NULL,
synthetic VARCHAR(192) NOT NULL,
userid BIGINT NOT NULL,
resp VARCHAR(255),
done TINYINT NOT NULL DEFAULT 0,
retries INT NOT NULL DEFAULT 0,
PRIMARY KEY(id),
FOREIGN KEY(userid) REFERENCES im_users(id),
UNIQUE credentials_uniqueness(synthetic)
)`); err != nil {
return err
}
// Records of uploaded files.
// Don't add FOREIGN KEY on userid. It's not needed and it will break user deletion.
if _, err = tx.Exec(
`CREATE TABLE im_fileuploads(
id BIGINT NOT NULL,
createdat DATETIME(3) NOT NULL,
updatedat DATETIME(3) NOT NULL,
userid BIGINT,
status INT NOT NULL,
mimetype VARCHAR(255) NOT NULL,
size BIGINT NOT NULL,
etag VARCHAR(128),
location VARCHAR(2048) NOT NULL,
PRIMARY KEY(id),
INDEX fileuploads_status(status)
)`); err != nil {
return err
}
// Links between uploaded files and the topics, users or messages they are attached to.
if _, err = tx.Exec(
`CREATE TABLE im_filemsglinks(
id INT NOT NULL AUTO_INCREMENT,
createdat DATETIME(3) NOT NULL,
fileid BIGINT NOT NULL,
msgid INT,
topic CHAR(25),
userid BIGINT,
PRIMARY KEY(id),
FOREIGN KEY(fileid) REFERENCES im_fileuploads(id) ON DELETE CASCADE,
FOREIGN KEY(msgid) REFERENCES im_messages(id) ON DELETE CASCADE,
FOREIGN KEY(topic) REFERENCES im_topics(name) ON DELETE CASCADE,
FOREIGN KEY(userid) REFERENCES im_users(id) ON DELETE CASCADE
)`); err != nil {
return err
}
if _, err = tx.Exec(
`CREATE TABLE im_kvmeta(` +
"id BIGINT NOT NULL," +
"`key` VARCHAR(64) NOT NULL," +
"createdat DATETIME(3)," +
"`value` TEXT," +
"PRIMARY KEY(id)," +
"UNIQUE INDEX uk_im_kvmeta_key(`key`)," +
"INDEX kvmeta_createdat_key(createdat, `key`)" +
`)`); err != nil {
return err
}
versionID, err := nextSnowflakeID()
if err != nil {
return err
}
if _, err = tx.Exec("INSERT INTO im_kvmeta(id, `key`, `value`) VALUES(?,'version',?)", versionID, adpVersion); err != nil {
return err
}
return tx.Commit()
return errors.New("mysql schema auto-creation is disabled; apply reviewed SQL manually before starting the service")
}
// UpgradeDb upgrades the database, if necessary.
// UpgradeDb is intentionally schema-read-only.
//
// Automatic ALTER/CREATE upgrade steps are disabled. If the database version
// differs from the adapter version, prepare and apply a reviewed SQL migration,
// then update im_kvmeta.version as part of that migration.
func (a *adapter) UpgradeDb() error {
if _, err := a.GetDbVersion(); err != nil {
return err
}
if err := a.ensureMessageTopicSeqIndex(); err != nil {
return err
}
if err := a.ensureTopicNameUniqueIndex(); err != nil {
return err
}
if err := a.ensureTopicSeqMatchesMessages(); err != nil {
return err
}
if a.version == adpVersion {
return nil
}
return errors.New("mysql independent-database multitenancy requires a fresh schema; automatic upgrade from version " +
strconv.Itoa(a.version) + " to " + strconv.Itoa(adpVersion) + " is not supported")
return errors.New("mysql schema auto-upgrade is disabled; apply reviewed SQL manually to upgrade from version " +
strconv.Itoa(a.version) + " to " + strconv.Itoa(adpVersion))
}
func (a *adapter) ensureMessageTopicSeqIndex() error {