feat(media): add MinIO storage switch and HTTPS OSS endpoint
This commit is contained in:
richjohnrambo
@@ -0,0 +1,29 @@
|
||||
# IM 文件存储切换（OSS / MinIO）
|
||||
|
||||
IM Server 继续提供 `/v0/file/u/` 上传和 `/v0/file/s/` 鉴权下载接口。客户端不直连对象存储。后台 `hotgo.hg_sys_config` 的 `im_third_party.media_handler` 是存储开关：`oss` 写阿里云 OSS，`minio` 写自建 MinIO。部署配置中的 `media.use_handler` 必须为 `sys_config`（`server/meetingim.conf` 已如此配置）；开关在服务启动时读取，修改后滚动重启 IM Server。
|
||||
|
||||
## 后台配置
|
||||
|
||||
先在 `hotgo` 执行 [IM-MinIO存储配置.sql](IM-MinIO存储配置.sql)，再通过后台填写：
|
||||
|
||||
| 配置 key | 值或用途 |
|
||||
| --- | --- |
|
||||
| `media_handler` | `oss` / `minio`，默认保留现有值 |
|
||||
| `minio_endpoint` | `https://minio.118im.com`；容器内如有可达的 HTTPS 内网地址，可改用内网地址 |
|
||||
| `minio_bucket` | `xgjs`，需预先创建 |
|
||||
| `minio_region` | `us-east-1` |
|
||||
| `minio_access_key_id` | MinIO Access Key，后台敏感配置 |
|
||||
| `minio_secret_access_key` | MinIO Secret Key，后台敏感配置 |
|
||||
|
||||
阿里云 OSS 的 `oss_*` 配置需继续保留，用于读取切换前的文件。回切 OSS 后，MinIO 的配置也需保留。新写入的 MinIO 对象在文件记录中保存为 `minio://<bucket>/<key>`；原有 OSS 文件位置不改。下载和清理按文件位置选择存储后端，不会因为切换开关迁移已有对象。
|
||||
|
||||
## 上线顺序
|
||||
|
||||
1. 确认 IM Server Pod 能访问 MinIO S3 API，且 `xgjs` 桶及访问密钥可用；不要使用 Console URL 作为 API endpoint。
|
||||
2. 执行配置 SQL 并在后台填写密钥；保留 `media_handler=oss`，发布包含 MinIO handler 的 IM Server。
|
||||
3. 将 `media_handler` 改为 `minio`，滚动重启 IM Server，在客户端做小文件上传、下载、Range 下载、删除和历史 OSS 文件下载验证。
|
||||
4. 回切时将 `media_handler` 改为 `oss` 并滚动重启；已有 MinIO 文件继续从 MinIO 读取。
|
||||
|
||||
MinIO 对象默认按私有文件处理。示例中的直连公网 URL 只有在桶策略允许公开读取时才可访问；IM 客户端应使用服务返回的文件 URL。当前阿里云内容安全接口接受阿里云 OSS 对象参数，MinIO 文件的第三方审核任务需另行接入可用的文件提交方式，不能把 MinIO 对象直接当 OSS 对象提交。
|
||||
|
||||
2026-10-17：本地和测试环境的 `hotgo` 均已加入 MinIO 字典与配置项，凭据已分别填入数据库，`media_handler` 仍为 `oss`。MinIO 公网 S3 API 的独立上传、读取、删除冒烟测试已通过；切换后仍需从测试环境 IM Server Pod 验证完整客户端上传链路。
|
||||
@@ -0,0 +1,44 @@
|
||||
-- 在 hotgo 后台库执行。只增加配置项，不切换当前存储后端。
|
||||
USE hotgo;
|
||||
|
||||
INSERT INTO hg_sys_dict_type
|
||||
(`pid`, `name`, `type`, `sort`, `remark`, `status`, `created_at`, `updated_at`)
|
||||
SELECT 0, 'IM 存储服务商', 'im_storage_provider', 210, 'IM 文件存储 provider 字典', 1, NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM hg_sys_dict_type WHERE `type` = 'im_storage_provider');
|
||||
|
||||
INSERT INTO hg_sys_dict_data
|
||||
(`label`, `value`, `value_type`, `type`, `list_class`, `is_default`, `sort`, `remark`, `status`, `created_at`, `updated_at`)
|
||||
SELECT 'MinIO', 'minio', 'string', 'im_storage_provider', 'info', 0, 20, '自建 MinIO 对象存储', 1, NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM hg_sys_dict_data WHERE `type` = 'im_storage_provider' AND `value` = 'minio');
|
||||
|
||||
INSERT INTO hg_sys_config
|
||||
(`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`)
|
||||
SELECT 'im_third_party', 'MinIO Endpoint', 'string', 'minio_endpoint', 'https://minio.118im.com', 'https://minio.118im.com', 22, 'MinIO S3 API 地址；优先使用内网 HTTPS 地址', 1, 1, NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'minio_endpoint');
|
||||
|
||||
INSERT INTO hg_sys_config
|
||||
(`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`)
|
||||
SELECT 'im_third_party', 'MinIO Bucket', 'string', 'minio_bucket', 'xgjs', 'xgjs', 23, 'MinIO 已创建的存储桶', 1, 1, NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'minio_bucket');
|
||||
|
||||
INSERT INTO hg_sys_config
|
||||
(`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`)
|
||||
SELECT 'im_third_party', 'MinIO Region', 'string', 'minio_region', 'us-east-1', 'us-east-1', 24, 'MinIO 签名地域', 1, 1, NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'minio_region');
|
||||
|
||||
INSERT INTO hg_sys_config
|
||||
(`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`)
|
||||
SELECT 'im_third_party', 'MinIO Access Key', 'string', 'minio_access_key_id', '', '', 25, '敏感配置；后台脱敏展示', 1, 1, NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'minio_access_key_id');
|
||||
|
||||
INSERT INTO hg_sys_config
|
||||
(`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`)
|
||||
SELECT 'im_third_party', 'MinIO Secret Key', 'string', 'minio_secret_access_key', '', '', 26, '敏感配置；后台脱敏展示', 1, 1, NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'minio_secret_access_key');
|
||||
|
||||
UPDATE hg_sys_config SET `tip`='IM 文件存储开关：oss 或 minio；修改后滚动重启 IM Server'
|
||||
WHERE `group`='im_third_party' AND `key`='media_handler';
|
||||
|
||||
-- 核验上传、下载、旧 OSS 文件读取后再执行：
|
||||
-- UPDATE hg_sys_config SET `value`='minio', updated_at=NOW()
|
||||
-- WHERE `group`='im_third_party' AND `key`='media_handler';
|
||||
@@ -45,6 +45,8 @@ tenants/{tenantID}/{yyyy}/{mm}/{fileID}
|
||||
| `verify_bucket` | 启动时是否联网校验 Bucket 存在 |
|
||||
| `use_sys_config` | 为 `true` 时从 `hotgo.hg_sys_config` 读取 OSS 配置 |
|
||||
|
||||
OSS endpoint 填写 `https://oss-cn-shenzhen.aliyuncs.com`。IM Server 初始化 OSS handler 时也会给裸域名补 HTTPS，并将阿里云 OSS 的显式 HTTP endpoint 改为 HTTPS。修改后台 `im_third_party.oss_endpoint` 后需重启 IM Server；发布后用真实文件上传验证 `POST /v0/file/u/`，确认 Pod 到 OSS 走 443 端口且不再出现 TCP 写超时。若 HTTPS 仍超时，继续检查 Pod 到 OSS 的出网链路。
|
||||
|
||||
当前 `server/tinode.conf` 仍默认使用 `fs`。切换 OSS 时，将 `media.use_handler` 改为 `oss`，并在 `media.handlers.oss` 中设置 `use_sys_config=true` 后，运行时会从后台库 `hotgo.hg_sys_config` 的 `im_third_party` 分组读取 OSS 配置。
|
||||
|
||||
代码入口：
|
||||
|
||||
@@ -142,7 +142,7 @@ WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' A
|
||||
|
||||
INSERT INTO hg_sys_config
|
||||
(`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`)
|
||||
SELECT 'im_third_party', 'OSS Endpoint', 'string', 'oss_endpoint', '', '', 12, '例如 oss-cn-shenzhen.aliyuncs.com，禁止写 AccessPoint endpoint', 1, 1, NOW(), NOW()
|
||||
SELECT 'im_third_party', 'OSS Endpoint', 'string', 'oss_endpoint', '', '', 12, '例如 https://oss-cn-shenzhen.aliyuncs.com；阿里云 OSS 上传使用 HTTPS，禁止写 AccessPoint endpoint', 1, 1, NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'oss_endpoint');
|
||||
|
||||
INSERT INTO hg_sys_config
|
||||
|
||||
@@ -24,6 +24,7 @@ require (
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/jmoiron/sqlx v1.4.0
|
||||
github.com/minio/minio-go/v7 v7.4.1
|
||||
github.com/mozillazg/go-pinyin v0.20.0
|
||||
github.com/nyaruka/phonenumbers v1.8.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
@@ -34,13 +35,13 @@ require (
|
||||
github.com/tinode/jsonco v1.0.0
|
||||
github.com/tinode/snowflake v1.0.0
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
golang.org/x/crypto v0.53.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sync v0.21.0
|
||||
golang.org/x/text v0.38.0
|
||||
golang.org/x/sync v0.23.0
|
||||
golang.org/x/text v0.42.0
|
||||
google.golang.org/api v0.287.0
|
||||
google.golang.org/grpc v1.82.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
google.golang.org/protobuf v1.36.12
|
||||
gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2
|
||||
)
|
||||
|
||||
@@ -80,6 +81,7 @@ require (
|
||||
github.com/clbanning/mxj/v2 v2.7.0 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
@@ -88,15 +90,22 @@ require (
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
|
||||
github.com/stretchr/objx v0.5.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/tjfoc/gmsm v1.4.1 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect
|
||||
@@ -106,8 +115,9 @@ require (
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
golang.org/x/net v0.56.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/net v0.60.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.3 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -129,18 +139,18 @@ require (
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/klauspost/compress v1.19.0 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/montanaflynn/stats v0.9.0 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/procfs v0.21.1 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/sirupsen/logrus v1.10.2 // indirect
|
||||
github.com/twilio/twilio-go v1.30.9
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.2.0 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/appengine v1.6.8 // indirect
|
||||
google.golang.org/genproto v0.0.0-20260630182238-925bb5da69e7 // indirect
|
||||
|
||||
@@ -176,6 +176,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=
|
||||
@@ -264,8 +266,13 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
|
||||
github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
@@ -281,6 +288,12 @@ github.com/localtunnel/go-localtunnel v0.0.0-20170326223115-8a804488f275 h1:IZyc
|
||||
github.com/localtunnel/go-localtunnel v0.0.0-20170326223115-8a804488f275/go.mod h1:zt6UU74K6Z6oMOYJbJzYpYucqdcQwSMPBEdSvGiaUMw=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w=
|
||||
github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@@ -304,6 +317,8 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
|
||||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
@@ -311,8 +326,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
@@ -326,13 +339,15 @@ github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgv
|
||||
github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
|
||||
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
|
||||
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
|
||||
github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
|
||||
github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
@@ -341,20 +356,26 @@ github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnO
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
|
||||
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/tinode/jsonco v1.0.0 h1:zVcpjzDvjuA1G+HLrckI5EiiRyq9jgV3x37OQl6e5FE=
|
||||
github.com/tinode/jsonco v1.0.0/go.mod h1:Bnavu3302Qfn2pILMNwASkelodgeew3IvDrbdzU84u8=
|
||||
github.com/tinode/snowflake v1.0.0 h1:YciQ9ZKn1TrnvpS8yZErt044XJaxWVtR9aMO9rOZVOE=
|
||||
github.com/tinode/snowflake v1.0.0/go.mod h1:5JiaCe3o7QdDeyRcAeZBGVghwRS+ygt2CF/hxmAoptQ=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w=
|
||||
github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho=
|
||||
github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE=
|
||||
@@ -372,6 +393,10 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
|
||||
github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
@@ -400,6 +425,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
@@ -415,8 +442,8 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf
|
||||
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
|
||||
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
|
||||
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
@@ -451,8 +478,8 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
|
||||
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
@@ -467,8 +494,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -492,8 +519,8 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -517,8 +544,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -569,8 +596,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
|
||||
gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII=
|
||||
gopkg.in/cenkalti/backoff.v2 v2.2.1/go.mod h1:S0QdOvT2AlerfSBkp0O+dk+bbIMaNbEmVk876gPCthU=
|
||||
@@ -581,8 +608,9 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
|
||||
gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
|
||||
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2 h1:tczPZjdz6soV2thcuq1IFOuNLrBUGonFyUXBbIWXWis=
|
||||
gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2/go.mod h1:c7Wo0IjB7JL9B9Avv0UZKorYJCUhiergpj3u1WtGT1E=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
@@ -590,7 +618,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
@@ -6,6 +6,7 @@ package mysql
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
|
||||
"github.com/tinode/chat/server/store"
|
||||
t "github.com/tinode/chat/server/store/types"
|
||||
@@ -16,6 +17,10 @@ type fileReviewExecContext interface {
|
||||
}
|
||||
|
||||
func insertFileReviewTasks(ctx context.Context, exec fileReviewExecContext, fd *t.FileDef) error {
|
||||
storageProvider := ""
|
||||
if strings.HasPrefix(fd.Location, "minio://") {
|
||||
storageProvider = "minio"
|
||||
}
|
||||
fileID := decodeUID(fd.Uid())
|
||||
userID := int64(0)
|
||||
if fd.User != "" {
|
||||
@@ -37,7 +42,7 @@ func insertFileReviewTasks(ctx context.Context, exec fileReviewExecContext, fd *
|
||||
_, err = exec.ExecContext(ctx, `INSERT IGNORE INTO im_file_review_tasks
|
||||
(id,task_id,file_id,user_id,task_type,provider,storage_provider,object_name,mime_type,file_size,status,created_at,updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
id, taskID, fileID, userID, task.taskType, task.provider, "", fd.Location, fd.MimeType, fd.Size,
|
||||
id, taskID, fileID, userID, task.taskType, task.provider, storageProvider, fd.Location, fd.MimeType, fd.Size,
|
||||
t.FileReviewStatusPending, fd.UpdatedAt, fd.UpdatedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+5
-4
@@ -281,17 +281,17 @@ func resolveMediaHandlerConfig(config *mediaConfig) (string, string, error) {
|
||||
conf = string(params)
|
||||
}
|
||||
}
|
||||
if handler == "oss" {
|
||||
ossConf, err := ensureOSSUsesSysConfig(conf)
|
||||
if handler == "oss" || handler == "minio" {
|
||||
storageConf, err := ensureObjectStorageUsesSysConfig(conf, handler)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return handler, ossConf, nil
|
||||
return handler, storageConf, nil
|
||||
}
|
||||
return handler, conf, nil
|
||||
}
|
||||
|
||||
func ensureOSSUsesSysConfig(conf string) (string, error) {
|
||||
func ensureObjectStorageUsesSysConfig(conf, provider string) (string, error) {
|
||||
params := map[string]any{}
|
||||
if strings.TrimSpace(conf) != "" {
|
||||
if err := json.Unmarshal([]byte(conf), ¶ms); err != nil {
|
||||
@@ -299,6 +299,7 @@ func ensureOSSUsesSysConfig(conf string) (string, error) {
|
||||
}
|
||||
}
|
||||
params["use_sys_config"] = true
|
||||
params["provider"] = provider
|
||||
encoded, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
|
||||
"github.com/tinode/chat/server/media"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
const minioLocationPrefix = "minio://"
|
||||
|
||||
type minioConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Bucket string `json:"bucket"`
|
||||
Region string `json:"region"`
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
SecretAccessKey string `json:"secret_access_key"`
|
||||
}
|
||||
|
||||
type minioStorage struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
func newMinIOStorage(conf minioConfig) (*minioStorage, error) {
|
||||
if conf.Endpoint == "" || conf.Bucket == "" || conf.AccessKeyID == "" || conf.SecretAccessKey == "" {
|
||||
return nil, nil
|
||||
}
|
||||
endpoint := strings.TrimSpace(conf.Endpoint)
|
||||
secure := true
|
||||
if strings.Contains(endpoint, "://") {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil || parsed.Host == "" || parsed.Path != "" || parsed.RawQuery != "" || parsed.Fragment != "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||||
return nil, errors.New("invalid MinIO endpoint")
|
||||
}
|
||||
endpoint = parsed.Host
|
||||
secure = parsed.Scheme == "https"
|
||||
}
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, ""),
|
||||
Secure: secure,
|
||||
Region: conf.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &minioStorage{client: client, bucket: conf.Bucket}, nil
|
||||
}
|
||||
|
||||
func (oh *ossHandler) uploadMinIO(fd *types.FileDef, file io.Reader) (string, int64, error) {
|
||||
key, err := tenantObjectKey(fd.TenantID, fd.Uid().String32(), fd.CreatedAt)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err := store.Files.StartUpload(fd.TenantID, fd); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
result, err := oh.minio.client.PutObject(context.Background(), oh.minio.bucket, key, file, remainingSize(file), minio.PutObjectOptions{
|
||||
ContentType: fd.MimeType,
|
||||
CacheControl: oh.conf.CacheControl,
|
||||
})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
fd.Location = minioLocation(oh.minio.bucket, key)
|
||||
fd.ETag = strings.Trim(result.ETag, "\"")
|
||||
return oh.conf.ServeURL + media.FileURLName(fd), result.Size, nil
|
||||
}
|
||||
|
||||
func (oh *ossHandler) uploadPublicMinIO(fd *media.PublicFileDef, file io.Reader) (string, int64, error) {
|
||||
key, err := publicObjectKey(fd.Purpose, fd.ID, fd.CreatedAt)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
result, err := oh.minio.client.PutObject(context.Background(), oh.minio.bucket, key, file, remainingSize(file), minio.PutObjectOptions{
|
||||
ContentType: fd.MimeType,
|
||||
CacheControl: oh.conf.CacheControl,
|
||||
})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
fd.Location = minioLocation(oh.minio.bucket, key)
|
||||
fd.ETag = strings.Trim(result.ETag, "\"")
|
||||
return fd.Location, result.Size, nil
|
||||
}
|
||||
|
||||
func (oh *ossHandler) downloadMinIO(fd *types.FileDef) (*types.FileDef, media.ReadSeekCloser, error) {
|
||||
bucket, key, err := oh.minioObject(fd.Location)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if fd.Size <= 0 {
|
||||
meta, err := oh.minio.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
fd.Size = meta.Size
|
||||
}
|
||||
reader := &ossReadSeekCloser{size: fd.Size}
|
||||
reader.open = func(offset int64) (io.ReadCloser, error) {
|
||||
options := minio.GetObjectOptions{}
|
||||
if offset > 0 {
|
||||
if err := options.SetRange(offset, fd.Size-1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return oh.minio.client.GetObject(context.Background(), bucket, key, options)
|
||||
}
|
||||
return fd, reader, nil
|
||||
}
|
||||
|
||||
func (oh *ossHandler) deleteMinIO(location string) error {
|
||||
bucket, key, err := oh.minioObject(location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return oh.minio.client.RemoveObject(context.Background(), bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (oh *ossHandler) minioObject(location string) (string, string, error) {
|
||||
if oh.minio == nil {
|
||||
return "", "", errors.New("MinIO is not configured for stored file")
|
||||
}
|
||||
bucket, key, ok := strings.Cut(strings.TrimPrefix(location, minioLocationPrefix), "/")
|
||||
if !strings.HasPrefix(location, minioLocationPrefix) || !ok || bucket != oh.minio.bucket || key == "" {
|
||||
return "", "", types.ErrMalformed
|
||||
}
|
||||
return bucket, key, nil
|
||||
}
|
||||
|
||||
func minioLocation(bucket, key string) string {
|
||||
return minioLocationPrefix + bucket + "/" + key
|
||||
}
|
||||
|
||||
func remainingSize(reader io.Reader) int64 {
|
||||
seeker, ok := reader.(io.Seeker)
|
||||
if !ok {
|
||||
return -1
|
||||
}
|
||||
current, err := seeker.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
end, err := seeker.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
_, _ = seeker.Seek(current, io.SeekStart)
|
||||
return -1
|
||||
}
|
||||
if _, err := seeker.Seek(current, io.SeekStart); err != nil {
|
||||
return -1
|
||||
}
|
||||
return end - current
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
func TestLiveMinIOUploadDownload(t *testing.T) {
|
||||
endpoint := os.Getenv("LIVE_MINIO_ENDPOINT")
|
||||
accessKey := os.Getenv("LIVE_MINIO_ACCESS_KEY")
|
||||
secretKey := os.Getenv("LIVE_MINIO_SECRET_KEY")
|
||||
bucket := os.Getenv("LIVE_MINIO_BUCKET")
|
||||
if endpoint == "" || accessKey == "" || secretKey == "" || bucket == "" {
|
||||
t.Skip("live MinIO configuration is not set")
|
||||
}
|
||||
storage, err := newMinIOStorage(minioConfig{
|
||||
Endpoint: endpoint, Bucket: bucket, Region: "us-east-1",
|
||||
AccessKeyID: accessKey, SecretAccessKey: secretKey,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
key := fmt.Sprintf("im-smoke/%d.txt", time.Now().UnixNano())
|
||||
defer func() {
|
||||
if err := storage.client.RemoveObject(context.Background(), bucket, key, minio.RemoveObjectOptions{}); err != nil {
|
||||
t.Errorf("cleanup object: %v", err)
|
||||
}
|
||||
}()
|
||||
content := []byte("im-minio-smoke-test")
|
||||
if _, err := storage.client.PutObject(ctx, bucket, key, bytes.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "text/plain"}); err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
object, err := storage.client.GetObject(ctx, bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
defer object.Close()
|
||||
got, err := io.ReadAll(object)
|
||||
if err != nil || !bytes.Equal(got, content) {
|
||||
t.Fatalf("downloaded content differs: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/media"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func TestMinIOUploadDownloadDelete(t *testing.T) {
|
||||
const body = "hello minio"
|
||||
var stored []byte
|
||||
var deleted bool
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/xgjs/public/") {
|
||||
t.Errorf("unexpected request path: %s", r.URL.Path)
|
||||
http.Error(w, "wrong path", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
stored, _ = io.ReadAll(r.Body)
|
||||
if strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") {
|
||||
separator := bytes.Index(stored, []byte("\r\n"))
|
||||
if separator < 0 || len(stored) < separator+2+len(body) {
|
||||
t.Errorf("invalid signed chunked request")
|
||||
http.Error(w, "invalid body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
stored = stored[separator+2 : separator+2+len(body)]
|
||||
}
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
case http.MethodGet:
|
||||
w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
if r.Header.Get("Range") == "bytes=6-10" {
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(stored[6:])
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(stored)
|
||||
case http.MethodDelete:
|
||||
deleted = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
t.Errorf("unexpected request method: %s", r.Method)
|
||||
http.Error(w, "wrong method", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
previous := store.ThirdPartyConfig
|
||||
store.ThirdPartyConfig = fakeThirdPartyConfig{
|
||||
"oss_enabled": "0",
|
||||
"minio_endpoint": server.URL,
|
||||
"minio_bucket": "xgjs",
|
||||
"minio_region": "us-east-1",
|
||||
"minio_access_key_id": "test-key",
|
||||
"minio_secret_access_key": "test-secret",
|
||||
}
|
||||
defer func() { store.ThirdPartyConfig = previous }()
|
||||
|
||||
handler := &ossHandler{}
|
||||
if err := handler.Init(`{"provider":"minio","use_sys_config":true}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if handler.bucket != nil || handler.minio == nil {
|
||||
t.Fatal("MinIO provider should initialize without OSS credentials")
|
||||
}
|
||||
file := &media.PublicFileDef{ID: "file_test", Purpose: "avatar", MimeType: "text/plain", CreatedAt: time.Now()}
|
||||
location, size, err := handler.UploadPublic(file, bytes.NewReader([]byte(body)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if size != int64(len(body)) || string(stored) != body || !strings.HasPrefix(location, "minio://xgjs/public/avatar/") {
|
||||
t.Fatalf("upload: location=%q size=%d data=%q", location, size, stored)
|
||||
}
|
||||
fd := &types.FileDef{Location: location, Size: size}
|
||||
_, reader, err := handler.downloadMinIO(fd)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
buf := make([]byte, len(body))
|
||||
if _, err := io.ReadFull(reader, buf); err != nil || string(buf) != body {
|
||||
t.Fatalf("download: %q, %v", buf, err)
|
||||
}
|
||||
if _, err := reader.Seek(6, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := io.ReadAll(reader); err != nil || string(got) != "minio" {
|
||||
t.Fatalf("range download: %q, %v", got, err)
|
||||
}
|
||||
if err := handler.Delete([]string{location}); err != nil || !deleted {
|
||||
t.Fatalf("delete: deleted=%v err=%v", deleted, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinIOLocationRejectsDifferentBucket(t *testing.T) {
|
||||
handler := &ossHandler{minio: &minioStorage{bucket: "xgjs"}}
|
||||
if _, _, err := handler.minioObject("minio://other/public/file"); err != types.ErrMalformed {
|
||||
t.Fatalf("wrong bucket should be rejected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinIOPublicEndpointUsesHTTPSPathStyle(t *testing.T) {
|
||||
storage, err := newMinIOStorage(minioConfig{
|
||||
Endpoint: "https://minio.118im.com", Bucket: "xgjs", Region: "us-east-1",
|
||||
AccessKeyID: "test-key", SecretAccessKey: "test-secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
url, err := storage.client.PresignedGetObject(context.Background(), storage.bucket, "images/test.jpg", time.Minute, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if url.Scheme != "https" || url.Host != "minio.118im.com" || url.Path != "/xgjs/images/test.jpg" {
|
||||
t.Fatalf("unexpected MinIO URL: %s", url)
|
||||
}
|
||||
}
|
||||
+102
-40
@@ -27,29 +27,33 @@ const (
|
||||
defaultServeURL = "/v0/file/s/"
|
||||
defaultCacheControl = "no-cache, must-revalidate"
|
||||
|
||||
handlerName = "oss"
|
||||
handlerName = "oss"
|
||||
minioHandlerName = "minio"
|
||||
// Presign GET URLs for this number of seconds.
|
||||
defaultPresignDuration = 120
|
||||
)
|
||||
|
||||
type ossConfig struct {
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
AccessKeySecret string `json:"access_key_secret"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
BucketName string `json:"bucket"`
|
||||
RegionID string `json:"region_id"`
|
||||
CorsOrigins []string `json:"cors_origins"`
|
||||
ServeURL string `json:"serve_url"`
|
||||
PresignTTL int `json:"presign_ttl"`
|
||||
CacheControl string `json:"cache_control"`
|
||||
VerifyBucket bool `json:"verify_bucket"`
|
||||
UseSysConfig bool `json:"use_sys_config"`
|
||||
Provider string `json:"provider"`
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
AccessKeySecret string `json:"access_key_secret"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
BucketName string `json:"bucket"`
|
||||
RegionID string `json:"region_id"`
|
||||
CorsOrigins []string `json:"cors_origins"`
|
||||
ServeURL string `json:"serve_url"`
|
||||
PresignTTL int `json:"presign_ttl"`
|
||||
CacheControl string `json:"cache_control"`
|
||||
VerifyBucket bool `json:"verify_bucket"`
|
||||
UseSysConfig bool `json:"use_sys_config"`
|
||||
MinIO minioConfig `json:"minio"`
|
||||
}
|
||||
|
||||
type ossHandler struct {
|
||||
client *aliyunoss.Client
|
||||
bucket *aliyunoss.Bucket
|
||||
conf ossConfig
|
||||
minio *minioStorage
|
||||
corsOrigins []media.AllowedOrigin
|
||||
}
|
||||
|
||||
@@ -138,21 +142,18 @@ func (oh *ossHandler) Init(jsconf string) error {
|
||||
if err = json.Unmarshal([]byte(jsconf), &oh.conf); err != nil {
|
||||
return errors.New("failed to parse config: " + err.Error())
|
||||
}
|
||||
if oh.conf.Provider == "" {
|
||||
oh.conf.Provider = handlerName
|
||||
}
|
||||
if oh.conf.Provider != handlerName && oh.conf.Provider != minioHandlerName {
|
||||
return errors.New("unsupported storage provider: " + oh.conf.Provider)
|
||||
}
|
||||
if err = oh.loadSysConfig(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if oh.conf.AccessKeyID == "" {
|
||||
return errors.New("missing Access Key ID")
|
||||
}
|
||||
if oh.conf.AccessKeySecret == "" {
|
||||
return errors.New("missing Access Key Secret")
|
||||
}
|
||||
if oh.conf.Endpoint == "" {
|
||||
return errors.New("missing Endpoint")
|
||||
}
|
||||
if oh.conf.BucketName == "" {
|
||||
return errors.New("missing Bucket")
|
||||
if oh.conf.Provider == handlerName && (oh.conf.AccessKeyID == "" || oh.conf.AccessKeySecret == "" || oh.conf.Endpoint == "" || oh.conf.BucketName == "") {
|
||||
return errors.New("incomplete OSS configuration")
|
||||
}
|
||||
if oh.conf.PresignTTL <= 0 {
|
||||
oh.conf.PresignTTL = defaultPresignDuration
|
||||
@@ -169,21 +170,46 @@ func (oh *ossHandler) Init(jsconf string) error {
|
||||
return errors.New("failed to parse CORS allowed origins: " + err.Error())
|
||||
}
|
||||
|
||||
oh.client, err = aliyunoss.New(oh.conf.Endpoint, oh.conf.AccessKeyID, oh.conf.AccessKeySecret)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if oh.conf.VerifyBucket {
|
||||
exists, err := oh.client.IsBucketExist(oh.conf.BucketName)
|
||||
if oh.conf.AccessKeyID != "" && oh.conf.AccessKeySecret != "" && oh.conf.Endpoint != "" && oh.conf.BucketName != "" {
|
||||
oh.conf.Endpoint = normalizeOSSEndpoint(oh.conf.Endpoint)
|
||||
oh.client, err = aliyunoss.New(oh.conf.Endpoint, oh.conf.AccessKeyID, oh.conf.AccessKeySecret)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return errors.New("bucket does not exist")
|
||||
if oh.conf.VerifyBucket && oh.conf.Provider == handlerName {
|
||||
exists, err := oh.client.IsBucketExist(oh.conf.BucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return errors.New("bucket does not exist")
|
||||
}
|
||||
}
|
||||
oh.bucket, err = oh.client.Bucket(oh.conf.BucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
oh.bucket, err = oh.client.Bucket(oh.conf.BucketName)
|
||||
return err
|
||||
oh.minio, err = newMinIOStorage(oh.conf.MinIO)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if oh.conf.Provider == minioHandlerName && oh.minio == nil {
|
||||
return errors.New("incomplete MinIO configuration")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeOSSEndpoint(endpoint string) string {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if !strings.Contains(endpoint, "://") {
|
||||
return "https://" + endpoint
|
||||
}
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err == nil && parsed.Scheme == "http" && strings.HasSuffix(strings.ToLower(parsed.Hostname()), ".aliyuncs.com") {
|
||||
return "https://" + strings.TrimPrefix(endpoint, "http://")
|
||||
}
|
||||
return endpoint
|
||||
}
|
||||
|
||||
func (oh *ossHandler) loadSysConfig() error {
|
||||
@@ -191,6 +217,11 @@ func (oh *ossHandler) loadSysConfig() error {
|
||||
return nil
|
||||
}
|
||||
values, err := store.ThirdPartyConfig.Get("im_third_party",
|
||||
"minio_endpoint",
|
||||
"minio_bucket",
|
||||
"minio_region",
|
||||
"minio_access_key_id",
|
||||
"minio_secret_access_key",
|
||||
"oss_enabled",
|
||||
"oss_provider",
|
||||
"oss_endpoint",
|
||||
@@ -207,12 +238,17 @@ func (oh *ossHandler) loadSysConfig() error {
|
||||
if err != nil {
|
||||
return errors.New("failed to load OSS config from sys config: " + err.Error())
|
||||
}
|
||||
if !truthy(values["oss_enabled"]) {
|
||||
if oh.conf.Provider == handlerName && !truthy(values["oss_enabled"]) {
|
||||
return errors.New("OSS is disabled in sys config")
|
||||
}
|
||||
if provider := strings.TrimSpace(values["oss_provider"]); provider != "" && provider != handlerName {
|
||||
if provider := strings.TrimSpace(values["oss_provider"]); oh.conf.Provider == handlerName && provider != "" && provider != handlerName {
|
||||
return errors.New("unsupported storage provider in sys config: " + provider)
|
||||
}
|
||||
applyString(&oh.conf.MinIO.Endpoint, values["minio_endpoint"])
|
||||
applyString(&oh.conf.MinIO.Bucket, values["minio_bucket"])
|
||||
applyString(&oh.conf.MinIO.Region, values["minio_region"])
|
||||
applyString(&oh.conf.MinIO.AccessKeyID, values["minio_access_key_id"])
|
||||
applyString(&oh.conf.MinIO.SecretAccessKey, values["minio_secret_access_key"])
|
||||
applyString(&oh.conf.Endpoint, values["oss_endpoint"])
|
||||
applyString(&oh.conf.BucketName, values["oss_bucket"])
|
||||
applyString(&oh.conf.RegionID, values["oss_region_id"])
|
||||
@@ -308,6 +344,9 @@ func (oh *ossHandler) Headers(tenantID types.TenantID, method string, url *url.U
|
||||
|
||||
// Upload processes request for a file upload.
|
||||
func (oh *ossHandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64, error) {
|
||||
if oh.conf.Provider == minioHandlerName {
|
||||
return oh.uploadMinIO(fdef, file)
|
||||
}
|
||||
key, err := tenantObjectKey(fdef.TenantID, fdef.Uid().String32(), fdef.CreatedAt)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
@@ -342,6 +381,9 @@ func (oh *ossHandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64
|
||||
}
|
||||
|
||||
func (oh *ossHandler) UploadPublic(fdef *media.PublicFileDef, file io.Reader) (string, int64, error) {
|
||||
if oh.conf.Provider == minioHandlerName {
|
||||
return oh.uploadPublicMinIO(fdef, file)
|
||||
}
|
||||
key, err := publicObjectKey(fdef.Purpose, fdef.ID, fdef.CreatedAt)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
@@ -378,6 +420,12 @@ func (oh *ossHandler) Download(tenantID types.TenantID, url string) (*types.File
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if strings.HasPrefix(fd.Location, minioLocationPrefix) {
|
||||
return oh.downloadMinIO(fd)
|
||||
}
|
||||
if oh.bucket == nil {
|
||||
return nil, nil, errors.New("OSS is not configured for legacy file download")
|
||||
}
|
||||
objectKey, err := oh.resolveObjectKey(fd.Location)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
@@ -405,12 +453,25 @@ func (oh *ossHandler) Download(tenantID types.TenantID, url string) (*types.File
|
||||
|
||||
// Delete deletes files from OSS by provided object keys.
|
||||
func (oh *ossHandler) Delete(locations []string) error {
|
||||
for i := 0; i < len(locations); i += 1000 {
|
||||
end := i + 1000
|
||||
if end > len(locations) {
|
||||
end = len(locations)
|
||||
ossLocations := make([]string, 0, len(locations))
|
||||
for _, location := range locations {
|
||||
if strings.HasPrefix(location, minioLocationPrefix) {
|
||||
if err := oh.deleteMinIO(location); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := oh.bucket.DeleteObjects(locations[i:end], aliyunoss.DeleteObjectsQuiet(true)); err != nil {
|
||||
ossLocations = append(ossLocations, location)
|
||||
}
|
||||
if len(ossLocations) > 0 && oh.bucket == nil {
|
||||
return errors.New("OSS is not configured for legacy file deletion")
|
||||
}
|
||||
for i := 0; i < len(ossLocations); i += 1000 {
|
||||
end := i + 1000
|
||||
if end > len(ossLocations) {
|
||||
end = len(ossLocations)
|
||||
}
|
||||
if _, err := oh.bucket.DeleteObjects(ossLocations[i:end], aliyunoss.DeleteObjectsQuiet(true)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -504,4 +565,5 @@ func etagFromLocation(location string) string {
|
||||
|
||||
func init() {
|
||||
store.RegisterMediaHandler(handlerName, &ossHandler{})
|
||||
store.RegisterMediaHandler(minioHandlerName, &ossHandler{})
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -123,3 +124,24 @@ func TestLoadSysConfig(t *testing.T) {
|
||||
t.Fatalf("unexpected OSS cors origins: %+v", handler.conf.CorsOrigins)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOSSEndpointUsesHTTPS(t *testing.T) {
|
||||
for _, endpoint := range []string{
|
||||
"oss-cn-shenzhen.aliyuncs.com",
|
||||
"http://oss-cn-shenzhen.aliyuncs.com",
|
||||
"https://oss-cn-shenzhen.aliyuncs.com",
|
||||
} {
|
||||
handler := &ossHandler{}
|
||||
config := `{"endpoint":"` + endpoint + `","access_key_id":"ak","access_key_secret":"sk","bucket":"im-files"}`
|
||||
if err := handler.Init(config); err != nil {
|
||||
t.Fatalf("Init(%q): %v", endpoint, err)
|
||||
}
|
||||
signedURL, err := handler.bucket.SignURL("test.txt", aliyunoss.HTTPGet, 60)
|
||||
if err != nil {
|
||||
t.Fatalf("SignURL(%q): %v", endpoint, err)
|
||||
}
|
||||
if !strings.HasPrefix(signedURL, "https://") {
|
||||
t.Fatalf("endpoint %q produced non-HTTPS URL %q", endpoint, signedURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/tinode/chat/server/store"
|
||||
)
|
||||
|
||||
type testMediaConfigProvider struct{ handler string }
|
||||
|
||||
func (p testMediaConfigProvider) Get(string, ...string) (map[string]string, error) {
|
||||
return map[string]string{"media_handler": p.handler}, nil
|
||||
}
|
||||
|
||||
func TestResolveObjectStorageMediaHandler(t *testing.T) {
|
||||
for _, provider := range []string{"oss", "minio"} {
|
||||
t.Run(provider, func(t *testing.T) {
|
||||
handler, raw, err := resolveMediaHandlerConfig(&mediaConfig{UseHandler: provider})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if handler != provider {
|
||||
t.Fatalf("handler = %q", handler)
|
||||
}
|
||||
var config struct {
|
||||
Provider string `json:"provider"`
|
||||
UseSysConfig bool `json:"use_sys_config"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &config); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Provider != provider || !config.UseSysConfig {
|
||||
t.Fatalf("resolved config = %+v", config)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveMediaHandlerFromSysConfig(t *testing.T) {
|
||||
previous := store.ThirdPartyConfig
|
||||
store.ThirdPartyConfig = testMediaConfigProvider{handler: "minio"}
|
||||
defer func() { store.ThirdPartyConfig = previous }()
|
||||
|
||||
handler, raw, err := resolveMediaHandlerConfig(&mediaConfig{UseHandler: "sys_config"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var config struct {
|
||||
Provider string `json:"provider"`
|
||||
UseSysConfig bool `json:"use_sys_config"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &config); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if handler != "minio" || config.Provider != "minio" || !config.UseSysConfig {
|
||||
t.Fatalf("handler=%q config=%+v", handler, config)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user