From 73c3dc0541596e9783e725b0d81f5d0ea87c24e0 Mon Sep 17 00:00:00 2001 From: richjohnrambo Date: Sat, 17 Oct 2026 22:34:34 +0800 Subject: [PATCH] feat(media): add MinIO storage switch and HTTPS OSS endpoint --- docs/third/IM-MinIO存储切换说明.md | 29 ++++ docs/third/IM-MinIO存储配置.sql | 44 ++++++ docs/third/IM-文件安全第三方接入记录.md | 2 + docs/third/IM-第三方配置与文件安全SQL.sql | 2 +- go.mod | 30 ++-- go.sum | 75 ++++++---- server/db/mysql/file_review_tasks.go | 7 +- server/main.go | 9 +- server/media/oss/minio.go | 161 ++++++++++++++++++++++ server/media/oss/minio_live_test.go | 51 +++++++ server/media/oss/minio_test.go | 129 +++++++++++++++++ server/media/oss/oss.go | 142 +++++++++++++------ server/media/oss/oss_tenant_test.go | 22 +++ server/media_config_test.go | 59 ++++++++ 14 files changed, 682 insertions(+), 80 deletions(-) create mode 100644 docs/third/IM-MinIO存储切换说明.md create mode 100644 docs/third/IM-MinIO存储配置.sql create mode 100644 server/media/oss/minio.go create mode 100644 server/media/oss/minio_live_test.go create mode 100644 server/media/oss/minio_test.go create mode 100644 server/media_config_test.go diff --git a/docs/third/IM-MinIO存储切换说明.md b/docs/third/IM-MinIO存储切换说明.md new file mode 100644 index 0000000..2e51af7 --- /dev/null +++ b/docs/third/IM-MinIO存储切换说明.md @@ -0,0 +1,29 @@ +# IM 文件存储切换（OSS / MinIO） + +IM Server 继续提供 `/v0/file/u/` 上传和 `/v0/file/s/` 鉴权下载接口。客户端不直连对象存储。后台 `hotgo.hg_sys_config` 的 `im_third_party.media_handler` 是存储开关：`oss` 写阿里云 OSS，`minio` 写自建 MinIO。部署配置中的 `media.use_handler` 必须为 `sys_config`（`server/meetingim.conf` 已如此配置）；开关在服务启动时读取，修改后滚动重启 IM Server。 + +## 后台配置 + +先在 `hotgo` 执行 [IM-MinIO存储配置.sql](IM-MinIO存储配置.sql)，再通过后台填写： + +| 配置 key | 值或用途 | +| --- | --- | +| `media_handler` | `oss` / `minio`，默认保留现有值 | +| `minio_endpoint` | `https://minio.118im.com`；容器内如有可达的 HTTPS 内网地址，可改用内网地址 | +| `minio_bucket` | `xgjs`，需预先创建 | +| `minio_region` | `us-east-1` | +| `minio_access_key_id` | MinIO Access Key，后台敏感配置 | +| `minio_secret_access_key` | MinIO Secret Key，后台敏感配置 | + +阿里云 OSS 的 `oss_*` 配置需继续保留，用于读取切换前的文件。回切 OSS 后，MinIO 的配置也需保留。新写入的 MinIO 对象在文件记录中保存为 `minio:///`；原有 OSS 文件位置不改。下载和清理按文件位置选择存储后端，不会因为切换开关迁移已有对象。 + +## 上线顺序 + +1. 确认 IM Server Pod 能访问 MinIO S3 API，且 `xgjs` 桶及访问密钥可用；不要使用 Console URL 作为 API endpoint。 +2. 执行配置 SQL 并在后台填写密钥；保留 `media_handler=oss`，发布包含 MinIO handler 的 IM Server。 +3. 将 `media_handler` 改为 `minio`，滚动重启 IM Server，在客户端做小文件上传、下载、Range 下载、删除和历史 OSS 文件下载验证。 +4. 回切时将 `media_handler` 改为 `oss` 并滚动重启；已有 MinIO 文件继续从 MinIO 读取。 + +MinIO 对象默认按私有文件处理。示例中的直连公网 URL 只有在桶策略允许公开读取时才可访问；IM 客户端应使用服务返回的文件 URL。当前阿里云内容安全接口接受阿里云 OSS 对象参数，MinIO 文件的第三方审核任务需另行接入可用的文件提交方式，不能把 MinIO 对象直接当 OSS 对象提交。 + +2026-10-17：本地和测试环境的 `hotgo` 均已加入 MinIO 字典与配置项，凭据已分别填入数据库，`media_handler` 仍为 `oss`。MinIO 公网 S3 API 的独立上传、读取、删除冒烟测试已通过；切换后仍需从测试环境 IM Server Pod 验证完整客户端上传链路。 diff --git a/docs/third/IM-MinIO存储配置.sql b/docs/third/IM-MinIO存储配置.sql new file mode 100644 index 0000000..9c8f570 --- /dev/null +++ b/docs/third/IM-MinIO存储配置.sql @@ -0,0 +1,44 @@ +-- 在 hotgo 后台库执行。只增加配置项，不切换当前存储后端。 +USE hotgo; + +INSERT INTO hg_sys_dict_type + (`pid`, `name`, `type`, `sort`, `remark`, `status`, `created_at`, `updated_at`) +SELECT 0, 'IM 存储服务商', 'im_storage_provider', 210, 'IM 文件存储 provider 字典', 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_dict_type WHERE `type` = 'im_storage_provider'); + +INSERT INTO hg_sys_dict_data + (`label`, `value`, `value_type`, `type`, `list_class`, `is_default`, `sort`, `remark`, `status`, `created_at`, `updated_at`) +SELECT 'MinIO', 'minio', 'string', 'im_storage_provider', 'info', 0, 20, '自建 MinIO 对象存储', 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_dict_data WHERE `type` = 'im_storage_provider' AND `value` = 'minio'); + +INSERT INTO hg_sys_config + (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) +SELECT 'im_third_party', 'MinIO Endpoint', 'string', 'minio_endpoint', 'https://minio.118im.com', 'https://minio.118im.com', 22, 'MinIO S3 API 地址；优先使用内网 HTTPS 地址', 1, 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'minio_endpoint'); + +INSERT INTO hg_sys_config + (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) +SELECT 'im_third_party', 'MinIO Bucket', 'string', 'minio_bucket', 'xgjs', 'xgjs', 23, 'MinIO 已创建的存储桶', 1, 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'minio_bucket'); + +INSERT INTO hg_sys_config + (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) +SELECT 'im_third_party', 'MinIO Region', 'string', 'minio_region', 'us-east-1', 'us-east-1', 24, 'MinIO 签名地域', 1, 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'minio_region'); + +INSERT INTO hg_sys_config + (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) +SELECT 'im_third_party', 'MinIO Access Key', 'string', 'minio_access_key_id', '', '', 25, '敏感配置；后台脱敏展示', 1, 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'minio_access_key_id'); + +INSERT INTO hg_sys_config + (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) +SELECT 'im_third_party', 'MinIO Secret Key', 'string', 'minio_secret_access_key', '', '', 26, '敏感配置；后台脱敏展示', 1, 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'minio_secret_access_key'); + +UPDATE hg_sys_config SET `tip`='IM 文件存储开关：oss 或 minio；修改后滚动重启 IM Server' +WHERE `group`='im_third_party' AND `key`='media_handler'; + +-- 核验上传、下载、旧 OSS 文件读取后再执行： +-- UPDATE hg_sys_config SET `value`='minio', updated_at=NOW() +-- WHERE `group`='im_third_party' AND `key`='media_handler'; diff --git a/docs/third/IM-文件安全第三方接入记录.md b/docs/third/IM-文件安全第三方接入记录.md index a001fc2..4bec101 100644 --- a/docs/third/IM-文件安全第三方接入记录.md +++ b/docs/third/IM-文件安全第三方接入记录.md @@ -45,6 +45,8 @@ tenants/{tenantID}/{yyyy}/{mm}/{fileID} | `verify_bucket` | 启动时是否联网校验 Bucket 存在 | | `use_sys_config` | 为 `true` 时从 `hotgo.hg_sys_config` 读取 OSS 配置 | +OSS endpoint 填写 `https://oss-cn-shenzhen.aliyuncs.com`。IM Server 初始化 OSS handler 时也会给裸域名补 HTTPS，并将阿里云 OSS 的显式 HTTP endpoint 改为 HTTPS。修改后台 `im_third_party.oss_endpoint` 后需重启 IM Server；发布后用真实文件上传验证 `POST /v0/file/u/`，确认 Pod 到 OSS 走 443 端口且不再出现 TCP 写超时。若 HTTPS 仍超时，继续检查 Pod 到 OSS 的出网链路。 + 当前 `server/tinode.conf` 仍默认使用 `fs`。切换 OSS 时，将 `media.use_handler` 改为 `oss`，并在 `media.handlers.oss` 中设置 `use_sys_config=true` 后，运行时会从后台库 `hotgo.hg_sys_config` 的 `im_third_party` 分组读取 OSS 配置。 代码入口： diff --git a/docs/third/IM-第三方配置与文件安全SQL.sql b/docs/third/IM-第三方配置与文件安全SQL.sql index b8b9374..93dca7d 100644 --- a/docs/third/IM-第三方配置与文件安全SQL.sql +++ b/docs/third/IM-第三方配置与文件安全SQL.sql @@ -142,7 +142,7 @@ WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' A INSERT INTO hg_sys_config (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) -SELECT 'im_third_party', 'OSS Endpoint', 'string', 'oss_endpoint', '', '', 12, '例如 oss-cn-shenzhen.aliyuncs.com，禁止写 AccessPoint endpoint', 1, 1, NOW(), NOW() +SELECT 'im_third_party', 'OSS Endpoint', 'string', 'oss_endpoint', '', '', 12, '例如 https://oss-cn-shenzhen.aliyuncs.com；阿里云 OSS 上传使用 HTTPS，禁止写 AccessPoint endpoint', 1, 1, NOW(), NOW() WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'oss_endpoint'); INSERT INTO hg_sys_config diff --git a/go.mod b/go.mod index 62575dd..8b6bb92 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/jackc/pgx/v5 v5.10.0 github.com/jmoiron/sqlx v1.4.0 + github.com/minio/minio-go/v7 v7.4.1 github.com/mozillazg/go-pinyin v0.20.0 github.com/nyaruka/phonenumbers v1.8.0 github.com/prometheus/client_golang v1.23.2 @@ -34,13 +35,13 @@ require ( github.com/tinode/jsonco v1.0.0 github.com/tinode/snowflake v1.0.0 go.mongodb.org/mongo-driver v1.17.9 - golang.org/x/crypto v0.53.0 + golang.org/x/crypto v0.57.0 golang.org/x/oauth2 v0.36.0 - golang.org/x/sync v0.21.0 - golang.org/x/text v0.38.0 + golang.org/x/sync v0.23.0 + golang.org/x/text v0.42.0 google.golang.org/api v0.287.0 google.golang.org/grpc v1.82.0 - google.golang.org/protobuf v1.36.11 + google.golang.org/protobuf v1.36.12 gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2 ) @@ -80,6 +81,7 @@ require ( github.com/clbanning/mxj/v2 v2.7.0 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect @@ -88,15 +90,22 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect - github.com/stretchr/objx v0.5.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect github.com/tjfoc/gmsm v1.4.1 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect @@ -106,8 +115,9 @@ require ( go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect - golang.org/x/net v0.56.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.60.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect ) require ( @@ -129,18 +139,18 @@ require ( github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/klauspost/compress v1.19.0 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/montanaflynn/stats v0.9.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/sirupsen/logrus v1.9.4 // indirect + github.com/sirupsen/logrus v1.10.2 // indirect github.com/twilio/twilio-go v1.30.9 github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - golang.org/x/sys v0.46.0 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20260630182238-925bb5da69e7 // indirect diff --git a/go.sum b/go.sum index 097c2c1..0fbf178 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= @@ -264,8 +266,13 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -281,6 +288,12 @@ github.com/localtunnel/go-localtunnel v0.0.0-20170326223115-8a804488f275 h1:IZyc github.com/localtunnel/go-localtunnel v0.0.0-20170326223115-8a804488f275/go.mod h1:zt6UU74K6Z6oMOYJbJzYpYucqdcQwSMPBEdSvGiaUMw= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= +github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -304,6 +317,8 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -311,8 +326,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -326,13 +339,15 @@ github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgv github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk= github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= -github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= +github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -341,20 +356,26 @@ github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnO github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/tinode/jsonco v1.0.0 h1:zVcpjzDvjuA1G+HLrckI5EiiRyq9jgV3x37OQl6e5FE= github.com/tinode/jsonco v1.0.0/go.mod h1:Bnavu3302Qfn2pILMNwASkelodgeew3IvDrbdzU84u8= github.com/tinode/snowflake v1.0.0 h1:YciQ9ZKn1TrnvpS8yZErt044XJaxWVtR9aMO9rOZVOE= github.com/tinode/snowflake v1.0.0/go.mod h1:5JiaCe3o7QdDeyRcAeZBGVghwRS+ygt2CF/hxmAoptQ= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= @@ -372,6 +393,10 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU= go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -400,6 +425,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -415,8 +442,8 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -451,8 +478,8 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= @@ -467,8 +494,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -492,8 +519,8 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -517,8 +544,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= @@ -569,8 +596,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII= gopkg.in/cenkalti/backoff.v2 v2.2.1/go.mod h1:S0QdOvT2AlerfSBkp0O+dk+bbIMaNbEmVk876gPCthU= @@ -581,8 +608,9 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2 h1:tczPZjdz6soV2thcuq1IFOuNLrBUGonFyUXBbIWXWis= gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2/go.mod h1:c7Wo0IjB7JL9B9Avv0UZKorYJCUhiergpj3u1WtGT1E= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -590,7 +618,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/server/db/mysql/file_review_tasks.go b/server/db/mysql/file_review_tasks.go index 23bd461..35084b8 100644 --- a/server/db/mysql/file_review_tasks.go +++ b/server/db/mysql/file_review_tasks.go @@ -6,6 +6,7 @@ package mysql import ( "context" "database/sql" + "strings" "github.com/tinode/chat/server/store" t "github.com/tinode/chat/server/store/types" @@ -16,6 +17,10 @@ type fileReviewExecContext interface { } func insertFileReviewTasks(ctx context.Context, exec fileReviewExecContext, fd *t.FileDef) error { + storageProvider := "" + if strings.HasPrefix(fd.Location, "minio://") { + storageProvider = "minio" + } fileID := decodeUID(fd.Uid()) userID := int64(0) if fd.User != "" { @@ -37,7 +42,7 @@ func insertFileReviewTasks(ctx context.Context, exec fileReviewExecContext, fd * _, err = exec.ExecContext(ctx, `INSERT IGNORE INTO im_file_review_tasks (id,task_id,file_id,user_id,task_type,provider,storage_provider,object_name,mime_type,file_size,status,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`, - id, taskID, fileID, userID, task.taskType, task.provider, "", fd.Location, fd.MimeType, fd.Size, + id, taskID, fileID, userID, task.taskType, task.provider, storageProvider, fd.Location, fd.MimeType, fd.Size, t.FileReviewStatusPending, fd.UpdatedAt, fd.UpdatedAt) if err != nil { return err diff --git a/server/main.go b/server/main.go index 7b2c6af..243a599 100644 --- a/server/main.go +++ b/server/main.go @@ -281,17 +281,17 @@ func resolveMediaHandlerConfig(config *mediaConfig) (string, string, error) { conf = string(params) } } - if handler == "oss" { - ossConf, err := ensureOSSUsesSysConfig(conf) + if handler == "oss" || handler == "minio" { + storageConf, err := ensureObjectStorageUsesSysConfig(conf, handler) if err != nil { return "", "", err } - return handler, ossConf, nil + return handler, storageConf, nil } return handler, conf, nil } -func ensureOSSUsesSysConfig(conf string) (string, error) { +func ensureObjectStorageUsesSysConfig(conf, provider string) (string, error) { params := map[string]any{} if strings.TrimSpace(conf) != "" { if err := json.Unmarshal([]byte(conf), ¶ms); err != nil { @@ -299,6 +299,7 @@ func ensureOSSUsesSysConfig(conf string) (string, error) { } } params["use_sys_config"] = true + params["provider"] = provider encoded, err := json.Marshal(params) if err != nil { return "", err diff --git a/server/media/oss/minio.go b/server/media/oss/minio.go new file mode 100644 index 0000000..4704736 --- /dev/null +++ b/server/media/oss/minio.go @@ -0,0 +1,161 @@ +package oss + +import ( + "context" + "errors" + "io" + "net/url" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + "github.com/tinode/chat/server/media" + "github.com/tinode/chat/server/store" + "github.com/tinode/chat/server/store/types" +) + +const minioLocationPrefix = "minio://" + +type minioConfig struct { + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` + Region string `json:"region"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` +} + +type minioStorage struct { + client *minio.Client + bucket string +} + +func newMinIOStorage(conf minioConfig) (*minioStorage, error) { + if conf.Endpoint == "" || conf.Bucket == "" || conf.AccessKeyID == "" || conf.SecretAccessKey == "" { + return nil, nil + } + endpoint := strings.TrimSpace(conf.Endpoint) + secure := true + if strings.Contains(endpoint, "://") { + parsed, err := url.Parse(endpoint) + if err != nil || parsed.Host == "" || parsed.Path != "" || parsed.RawQuery != "" || parsed.Fragment != "" || (parsed.Scheme != "http" && parsed.Scheme != "https") { + return nil, errors.New("invalid MinIO endpoint") + } + endpoint = parsed.Host + secure = parsed.Scheme == "https" + } + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, ""), + Secure: secure, + Region: conf.Region, + }) + if err != nil { + return nil, err + } + return &minioStorage{client: client, bucket: conf.Bucket}, nil +} + +func (oh *ossHandler) uploadMinIO(fd *types.FileDef, file io.Reader) (string, int64, error) { + key, err := tenantObjectKey(fd.TenantID, fd.Uid().String32(), fd.CreatedAt) + if err != nil { + return "", 0, err + } + if err := store.Files.StartUpload(fd.TenantID, fd); err != nil { + return "", 0, err + } + result, err := oh.minio.client.PutObject(context.Background(), oh.minio.bucket, key, file, remainingSize(file), minio.PutObjectOptions{ + ContentType: fd.MimeType, + CacheControl: oh.conf.CacheControl, + }) + if err != nil { + return "", 0, err + } + fd.Location = minioLocation(oh.minio.bucket, key) + fd.ETag = strings.Trim(result.ETag, "\"") + return oh.conf.ServeURL + media.FileURLName(fd), result.Size, nil +} + +func (oh *ossHandler) uploadPublicMinIO(fd *media.PublicFileDef, file io.Reader) (string, int64, error) { + key, err := publicObjectKey(fd.Purpose, fd.ID, fd.CreatedAt) + if err != nil { + return "", 0, err + } + result, err := oh.minio.client.PutObject(context.Background(), oh.minio.bucket, key, file, remainingSize(file), minio.PutObjectOptions{ + ContentType: fd.MimeType, + CacheControl: oh.conf.CacheControl, + }) + if err != nil { + return "", 0, err + } + fd.Location = minioLocation(oh.minio.bucket, key) + fd.ETag = strings.Trim(result.ETag, "\"") + return fd.Location, result.Size, nil +} + +func (oh *ossHandler) downloadMinIO(fd *types.FileDef) (*types.FileDef, media.ReadSeekCloser, error) { + bucket, key, err := oh.minioObject(fd.Location) + if err != nil { + return nil, nil, err + } + if fd.Size <= 0 { + meta, err := oh.minio.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{}) + if err != nil { + return nil, nil, err + } + fd.Size = meta.Size + } + reader := &ossReadSeekCloser{size: fd.Size} + reader.open = func(offset int64) (io.ReadCloser, error) { + options := minio.GetObjectOptions{} + if offset > 0 { + if err := options.SetRange(offset, fd.Size-1); err != nil { + return nil, err + } + } + return oh.minio.client.GetObject(context.Background(), bucket, key, options) + } + return fd, reader, nil +} + +func (oh *ossHandler) deleteMinIO(location string) error { + bucket, key, err := oh.minioObject(location) + if err != nil { + return err + } + return oh.minio.client.RemoveObject(context.Background(), bucket, key, minio.RemoveObjectOptions{}) +} + +func (oh *ossHandler) minioObject(location string) (string, string, error) { + if oh.minio == nil { + return "", "", errors.New("MinIO is not configured for stored file") + } + bucket, key, ok := strings.Cut(strings.TrimPrefix(location, minioLocationPrefix), "/") + if !strings.HasPrefix(location, minioLocationPrefix) || !ok || bucket != oh.minio.bucket || key == "" { + return "", "", types.ErrMalformed + } + return bucket, key, nil +} + +func minioLocation(bucket, key string) string { + return minioLocationPrefix + bucket + "/" + key +} + +func remainingSize(reader io.Reader) int64 { + seeker, ok := reader.(io.Seeker) + if !ok { + return -1 + } + current, err := seeker.Seek(0, io.SeekCurrent) + if err != nil { + return -1 + } + end, err := seeker.Seek(0, io.SeekEnd) + if err != nil { + _, _ = seeker.Seek(current, io.SeekStart) + return -1 + } + if _, err := seeker.Seek(current, io.SeekStart); err != nil { + return -1 + } + return end - current +} diff --git a/server/media/oss/minio_live_test.go b/server/media/oss/minio_live_test.go new file mode 100644 index 0000000..5f15dfc --- /dev/null +++ b/server/media/oss/minio_live_test.go @@ -0,0 +1,51 @@ +package oss + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "testing" + "time" + + "github.com/minio/minio-go/v7" +) + +func TestLiveMinIOUploadDownload(t *testing.T) { + endpoint := os.Getenv("LIVE_MINIO_ENDPOINT") + accessKey := os.Getenv("LIVE_MINIO_ACCESS_KEY") + secretKey := os.Getenv("LIVE_MINIO_SECRET_KEY") + bucket := os.Getenv("LIVE_MINIO_BUCKET") + if endpoint == "" || accessKey == "" || secretKey == "" || bucket == "" { + t.Skip("live MinIO configuration is not set") + } + storage, err := newMinIOStorage(minioConfig{ + Endpoint: endpoint, Bucket: bucket, Region: "us-east-1", + AccessKeyID: accessKey, SecretAccessKey: secretKey, + }) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + key := fmt.Sprintf("im-smoke/%d.txt", time.Now().UnixNano()) + defer func() { + if err := storage.client.RemoveObject(context.Background(), bucket, key, minio.RemoveObjectOptions{}); err != nil { + t.Errorf("cleanup object: %v", err) + } + }() + content := []byte("im-minio-smoke-test") + if _, err := storage.client.PutObject(ctx, bucket, key, bytes.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "text/plain"}); err != nil { + t.Fatalf("upload: %v", err) + } + object, err := storage.client.GetObject(ctx, bucket, key, minio.GetObjectOptions{}) + if err != nil { + t.Fatalf("download: %v", err) + } + defer object.Close() + got, err := io.ReadAll(object) + if err != nil || !bytes.Equal(got, content) { + t.Fatalf("downloaded content differs: %v", err) + } +} diff --git a/server/media/oss/minio_test.go b/server/media/oss/minio_test.go new file mode 100644 index 0000000..aefe1b9 --- /dev/null +++ b/server/media/oss/minio_test.go @@ -0,0 +1,129 @@ +package oss + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/tinode/chat/server/media" + "github.com/tinode/chat/server/store" + "github.com/tinode/chat/server/store/types" +) + +func TestMinIOUploadDownloadDelete(t *testing.T) { + const body = "hello minio" + var stored []byte + var deleted bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.URL.Path, "/xgjs/public/") { + t.Errorf("unexpected request path: %s", r.URL.Path) + http.Error(w, "wrong path", http.StatusNotFound) + return + } + switch r.Method { + case http.MethodPut: + stored, _ = io.ReadAll(r.Body) + if strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") { + separator := bytes.Index(stored, []byte("\r\n")) + if separator < 0 || len(stored) < separator+2+len(body) { + t.Errorf("invalid signed chunked request") + http.Error(w, "invalid body", http.StatusBadRequest) + return + } + stored = stored[separator+2 : separator+2+len(body)] + } + w.Header().Set("ETag", `"test-etag"`) + case http.MethodGet: + w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat)) + w.Header().Set("ETag", `"test-etag"`) + if r.Header.Get("Range") == "bytes=6-10" { + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(stored[6:]) + return + } + _, _ = w.Write(stored) + case http.MethodDelete: + deleted = true + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected request method: %s", r.Method) + http.Error(w, "wrong method", http.StatusMethodNotAllowed) + } + })) + defer server.Close() + + previous := store.ThirdPartyConfig + store.ThirdPartyConfig = fakeThirdPartyConfig{ + "oss_enabled": "0", + "minio_endpoint": server.URL, + "minio_bucket": "xgjs", + "minio_region": "us-east-1", + "minio_access_key_id": "test-key", + "minio_secret_access_key": "test-secret", + } + defer func() { store.ThirdPartyConfig = previous }() + + handler := &ossHandler{} + if err := handler.Init(`{"provider":"minio","use_sys_config":true}`); err != nil { + t.Fatal(err) + } + if handler.bucket != nil || handler.minio == nil { + t.Fatal("MinIO provider should initialize without OSS credentials") + } + file := &media.PublicFileDef{ID: "file_test", Purpose: "avatar", MimeType: "text/plain", CreatedAt: time.Now()} + location, size, err := handler.UploadPublic(file, bytes.NewReader([]byte(body))) + if err != nil { + t.Fatal(err) + } + if size != int64(len(body)) || string(stored) != body || !strings.HasPrefix(location, "minio://xgjs/public/avatar/") { + t.Fatalf("upload: location=%q size=%d data=%q", location, size, stored) + } + fd := &types.FileDef{Location: location, Size: size} + _, reader, err := handler.downloadMinIO(fd) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + buf := make([]byte, len(body)) + if _, err := io.ReadFull(reader, buf); err != nil || string(buf) != body { + t.Fatalf("download: %q, %v", buf, err) + } + if _, err := reader.Seek(6, io.SeekStart); err != nil { + t.Fatal(err) + } + if got, err := io.ReadAll(reader); err != nil || string(got) != "minio" { + t.Fatalf("range download: %q, %v", got, err) + } + if err := handler.Delete([]string{location}); err != nil || !deleted { + t.Fatalf("delete: deleted=%v err=%v", deleted, err) + } +} + +func TestMinIOLocationRejectsDifferentBucket(t *testing.T) { + handler := &ossHandler{minio: &minioStorage{bucket: "xgjs"}} + if _, _, err := handler.minioObject("minio://other/public/file"); err != types.ErrMalformed { + t.Fatalf("wrong bucket should be rejected, got %v", err) + } +} + +func TestMinIOPublicEndpointUsesHTTPSPathStyle(t *testing.T) { + storage, err := newMinIOStorage(minioConfig{ + Endpoint: "https://minio.118im.com", Bucket: "xgjs", Region: "us-east-1", + AccessKeyID: "test-key", SecretAccessKey: "test-secret", + }) + if err != nil { + t.Fatal(err) + } + url, err := storage.client.PresignedGetObject(context.Background(), storage.bucket, "images/test.jpg", time.Minute, nil) + if err != nil { + t.Fatal(err) + } + if url.Scheme != "https" || url.Host != "minio.118im.com" || url.Path != "/xgjs/images/test.jpg" { + t.Fatalf("unexpected MinIO URL: %s", url) + } +} diff --git a/server/media/oss/oss.go b/server/media/oss/oss.go index 56425b1..ee4ec8d 100644 --- a/server/media/oss/oss.go +++ b/server/media/oss/oss.go @@ -27,29 +27,33 @@ const ( defaultServeURL = "/v0/file/s/" defaultCacheControl = "no-cache, must-revalidate" - handlerName = "oss" + handlerName = "oss" + minioHandlerName = "minio" // Presign GET URLs for this number of seconds. defaultPresignDuration = 120 ) type ossConfig struct { - AccessKeyID string `json:"access_key_id"` - AccessKeySecret string `json:"access_key_secret"` - Endpoint string `json:"endpoint"` - BucketName string `json:"bucket"` - RegionID string `json:"region_id"` - CorsOrigins []string `json:"cors_origins"` - ServeURL string `json:"serve_url"` - PresignTTL int `json:"presign_ttl"` - CacheControl string `json:"cache_control"` - VerifyBucket bool `json:"verify_bucket"` - UseSysConfig bool `json:"use_sys_config"` + Provider string `json:"provider"` + AccessKeyID string `json:"access_key_id"` + AccessKeySecret string `json:"access_key_secret"` + Endpoint string `json:"endpoint"` + BucketName string `json:"bucket"` + RegionID string `json:"region_id"` + CorsOrigins []string `json:"cors_origins"` + ServeURL string `json:"serve_url"` + PresignTTL int `json:"presign_ttl"` + CacheControl string `json:"cache_control"` + VerifyBucket bool `json:"verify_bucket"` + UseSysConfig bool `json:"use_sys_config"` + MinIO minioConfig `json:"minio"` } type ossHandler struct { client *aliyunoss.Client bucket *aliyunoss.Bucket conf ossConfig + minio *minioStorage corsOrigins []media.AllowedOrigin } @@ -138,21 +142,18 @@ func (oh *ossHandler) Init(jsconf string) error { if err = json.Unmarshal([]byte(jsconf), &oh.conf); err != nil { return errors.New("failed to parse config: " + err.Error()) } + if oh.conf.Provider == "" { + oh.conf.Provider = handlerName + } + if oh.conf.Provider != handlerName && oh.conf.Provider != minioHandlerName { + return errors.New("unsupported storage provider: " + oh.conf.Provider) + } if err = oh.loadSysConfig(); err != nil { return err } - if oh.conf.AccessKeyID == "" { - return errors.New("missing Access Key ID") - } - if oh.conf.AccessKeySecret == "" { - return errors.New("missing Access Key Secret") - } - if oh.conf.Endpoint == "" { - return errors.New("missing Endpoint") - } - if oh.conf.BucketName == "" { - return errors.New("missing Bucket") + if oh.conf.Provider == handlerName && (oh.conf.AccessKeyID == "" || oh.conf.AccessKeySecret == "" || oh.conf.Endpoint == "" || oh.conf.BucketName == "") { + return errors.New("incomplete OSS configuration") } if oh.conf.PresignTTL <= 0 { oh.conf.PresignTTL = defaultPresignDuration @@ -169,21 +170,46 @@ func (oh *ossHandler) Init(jsconf string) error { return errors.New("failed to parse CORS allowed origins: " + err.Error()) } - oh.client, err = aliyunoss.New(oh.conf.Endpoint, oh.conf.AccessKeyID, oh.conf.AccessKeySecret) - if err != nil { - return err - } - if oh.conf.VerifyBucket { - exists, err := oh.client.IsBucketExist(oh.conf.BucketName) + if oh.conf.AccessKeyID != "" && oh.conf.AccessKeySecret != "" && oh.conf.Endpoint != "" && oh.conf.BucketName != "" { + oh.conf.Endpoint = normalizeOSSEndpoint(oh.conf.Endpoint) + oh.client, err = aliyunoss.New(oh.conf.Endpoint, oh.conf.AccessKeyID, oh.conf.AccessKeySecret) if err != nil { return err } - if !exists { - return errors.New("bucket does not exist") + if oh.conf.VerifyBucket && oh.conf.Provider == handlerName { + exists, err := oh.client.IsBucketExist(oh.conf.BucketName) + if err != nil { + return err + } + if !exists { + return errors.New("bucket does not exist") + } + } + oh.bucket, err = oh.client.Bucket(oh.conf.BucketName) + if err != nil { + return err } } - oh.bucket, err = oh.client.Bucket(oh.conf.BucketName) - return err + oh.minio, err = newMinIOStorage(oh.conf.MinIO) + if err != nil { + return err + } + if oh.conf.Provider == minioHandlerName && oh.minio == nil { + return errors.New("incomplete MinIO configuration") + } + return nil +} + +func normalizeOSSEndpoint(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + if !strings.Contains(endpoint, "://") { + return "https://" + endpoint + } + parsed, err := url.Parse(endpoint) + if err == nil && parsed.Scheme == "http" && strings.HasSuffix(strings.ToLower(parsed.Hostname()), ".aliyuncs.com") { + return "https://" + strings.TrimPrefix(endpoint, "http://") + } + return endpoint } func (oh *ossHandler) loadSysConfig() error { @@ -191,6 +217,11 @@ func (oh *ossHandler) loadSysConfig() error { return nil } values, err := store.ThirdPartyConfig.Get("im_third_party", + "minio_endpoint", + "minio_bucket", + "minio_region", + "minio_access_key_id", + "minio_secret_access_key", "oss_enabled", "oss_provider", "oss_endpoint", @@ -207,12 +238,17 @@ func (oh *ossHandler) loadSysConfig() error { if err != nil { return errors.New("failed to load OSS config from sys config: " + err.Error()) } - if !truthy(values["oss_enabled"]) { + if oh.conf.Provider == handlerName && !truthy(values["oss_enabled"]) { return errors.New("OSS is disabled in sys config") } - if provider := strings.TrimSpace(values["oss_provider"]); provider != "" && provider != handlerName { + if provider := strings.TrimSpace(values["oss_provider"]); oh.conf.Provider == handlerName && provider != "" && provider != handlerName { return errors.New("unsupported storage provider in sys config: " + provider) } + applyString(&oh.conf.MinIO.Endpoint, values["minio_endpoint"]) + applyString(&oh.conf.MinIO.Bucket, values["minio_bucket"]) + applyString(&oh.conf.MinIO.Region, values["minio_region"]) + applyString(&oh.conf.MinIO.AccessKeyID, values["minio_access_key_id"]) + applyString(&oh.conf.MinIO.SecretAccessKey, values["minio_secret_access_key"]) applyString(&oh.conf.Endpoint, values["oss_endpoint"]) applyString(&oh.conf.BucketName, values["oss_bucket"]) applyString(&oh.conf.RegionID, values["oss_region_id"]) @@ -308,6 +344,9 @@ func (oh *ossHandler) Headers(tenantID types.TenantID, method string, url *url.U // Upload processes request for a file upload. func (oh *ossHandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64, error) { + if oh.conf.Provider == minioHandlerName { + return oh.uploadMinIO(fdef, file) + } key, err := tenantObjectKey(fdef.TenantID, fdef.Uid().String32(), fdef.CreatedAt) if err != nil { return "", 0, err @@ -342,6 +381,9 @@ func (oh *ossHandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64 } func (oh *ossHandler) UploadPublic(fdef *media.PublicFileDef, file io.Reader) (string, int64, error) { + if oh.conf.Provider == minioHandlerName { + return oh.uploadPublicMinIO(fdef, file) + } key, err := publicObjectKey(fdef.Purpose, fdef.ID, fdef.CreatedAt) if err != nil { return "", 0, err @@ -378,6 +420,12 @@ func (oh *ossHandler) Download(tenantID types.TenantID, url string) (*types.File if err != nil { return nil, nil, err } + if strings.HasPrefix(fd.Location, minioLocationPrefix) { + return oh.downloadMinIO(fd) + } + if oh.bucket == nil { + return nil, nil, errors.New("OSS is not configured for legacy file download") + } objectKey, err := oh.resolveObjectKey(fd.Location) if err != nil { return nil, nil, err @@ -405,12 +453,25 @@ func (oh *ossHandler) Download(tenantID types.TenantID, url string) (*types.File // Delete deletes files from OSS by provided object keys. func (oh *ossHandler) Delete(locations []string) error { - for i := 0; i < len(locations); i += 1000 { - end := i + 1000 - if end > len(locations) { - end = len(locations) + ossLocations := make([]string, 0, len(locations)) + for _, location := range locations { + if strings.HasPrefix(location, minioLocationPrefix) { + if err := oh.deleteMinIO(location); err != nil { + return err + } + continue } - if _, err := oh.bucket.DeleteObjects(locations[i:end], aliyunoss.DeleteObjectsQuiet(true)); err != nil { + ossLocations = append(ossLocations, location) + } + if len(ossLocations) > 0 && oh.bucket == nil { + return errors.New("OSS is not configured for legacy file deletion") + } + for i := 0; i < len(ossLocations); i += 1000 { + end := i + 1000 + if end > len(ossLocations) { + end = len(ossLocations) + } + if _, err := oh.bucket.DeleteObjects(ossLocations[i:end], aliyunoss.DeleteObjectsQuiet(true)); err != nil { return err } } @@ -504,4 +565,5 @@ func etagFromLocation(location string) string { func init() { store.RegisterMediaHandler(handlerName, &ossHandler{}) + store.RegisterMediaHandler(minioHandlerName, &ossHandler{}) } diff --git a/server/media/oss/oss_tenant_test.go b/server/media/oss/oss_tenant_test.go index 9a5799b..9c077de 100644 --- a/server/media/oss/oss_tenant_test.go +++ b/server/media/oss/oss_tenant_test.go @@ -4,6 +4,7 @@ import ( "bytes" "io" "net/http" + "strings" "testing" "time" @@ -123,3 +124,24 @@ func TestLoadSysConfig(t *testing.T) { t.Fatalf("unexpected OSS cors origins: %+v", handler.conf.CorsOrigins) } } + +func TestOSSEndpointUsesHTTPS(t *testing.T) { + for _, endpoint := range []string{ + "oss-cn-shenzhen.aliyuncs.com", + "http://oss-cn-shenzhen.aliyuncs.com", + "https://oss-cn-shenzhen.aliyuncs.com", + } { + handler := &ossHandler{} + config := `{"endpoint":"` + endpoint + `","access_key_id":"ak","access_key_secret":"sk","bucket":"im-files"}` + if err := handler.Init(config); err != nil { + t.Fatalf("Init(%q): %v", endpoint, err) + } + signedURL, err := handler.bucket.SignURL("test.txt", aliyunoss.HTTPGet, 60) + if err != nil { + t.Fatalf("SignURL(%q): %v", endpoint, err) + } + if !strings.HasPrefix(signedURL, "https://") { + t.Fatalf("endpoint %q produced non-HTTPS URL %q", endpoint, signedURL) + } + } +} diff --git a/server/media_config_test.go b/server/media_config_test.go new file mode 100644 index 0000000..d7775a7 --- /dev/null +++ b/server/media_config_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "encoding/json" + "testing" + + "github.com/tinode/chat/server/store" +) + +type testMediaConfigProvider struct{ handler string } + +func (p testMediaConfigProvider) Get(string, ...string) (map[string]string, error) { + return map[string]string{"media_handler": p.handler}, nil +} + +func TestResolveObjectStorageMediaHandler(t *testing.T) { + for _, provider := range []string{"oss", "minio"} { + t.Run(provider, func(t *testing.T) { + handler, raw, err := resolveMediaHandlerConfig(&mediaConfig{UseHandler: provider}) + if err != nil { + t.Fatal(err) + } + if handler != provider { + t.Fatalf("handler = %q", handler) + } + var config struct { + Provider string `json:"provider"` + UseSysConfig bool `json:"use_sys_config"` + } + if err := json.Unmarshal([]byte(raw), &config); err != nil { + t.Fatal(err) + } + if config.Provider != provider || !config.UseSysConfig { + t.Fatalf("resolved config = %+v", config) + } + }) + } +} + +func TestResolveMediaHandlerFromSysConfig(t *testing.T) { + previous := store.ThirdPartyConfig + store.ThirdPartyConfig = testMediaConfigProvider{handler: "minio"} + defer func() { store.ThirdPartyConfig = previous }() + + handler, raw, err := resolveMediaHandlerConfig(&mediaConfig{UseHandler: "sys_config"}) + if err != nil { + t.Fatal(err) + } + var config struct { + Provider string `json:"provider"` + UseSysConfig bool `json:"use_sys_config"` + } + if err := json.Unmarshal([]byte(raw), &config); err != nil { + t.Fatal(err) + } + if handler != "minio" || config.Provider != "minio" || !config.UseSysConfig { + t.Fatalf("handler=%q config=%+v", handler, config) + } +}