chore: sync project files

server/.golangci.yml

@@ -0,0 +1,45 @@
+linters-settings:
+  govet:
+    check-shadowing: true
+    disable:
+      - composites
+  golint:
+    min-confidence: 0
+  gocyclo:
+    min-complexity: 10
+  maligned:
+    suggest-new: true
+  dupl:
+    threshold: 100
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  misspell:
+    locale: US
+  lll:
+    line-length: 140
+  gocritic:
+    enabled-tags:
+      - performance
+      - style
+      - experimental
+    disabled-checks:
+      - wrapperFunc
+      - commentFormatting # https://github.com/go-critic/go-critic/issues/755
+
+linters:
+  enable-all: true
+  disable:
+    - errcheck
+    - maligned
+    - prealloc
+    - gosec
+    - gochecknoglobals
+
+# options for analysis running
+run:
+  # list of build tags, all linters use it. Default is empty list.
+  build-tags:
+    - mysql
+    - rethinkdb
+    - mongodb

server/api_key.go

@@ -0,0 +1,74 @@
+/******************************************************************************
+ *
+ *  Description :
+ *
+ *  Authentication
+ *
+ *****************************************************************************/
+
+package main
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/md5"
+	"encoding/base64"
+
+	"github.com/tinode/chat/server/logs"
+)
+
+// Singned AppID. Composition:
+//
+//	[1:algorithm version][4:appid][2:key sequence][1:isRoot][16:signature] = 24 bytes
+//
+// convertible to base64 without padding. All integers are little-endian.
+// Definitions for byte lengths of key's parts.
+const (
+	// apikeyVersion is the version of this API scheme.
+	apikeyVersion = 1
+	// apikeyAppID is deprecated and will be removed in the future.
+	apikeyAppID = 4
+	// apikeySequence is the serial number of the key.
+	apikeySequence = 2
+	// apikeyWho indicates if the key grants root privileges.
+	apikeyWho = 1
+	// apikeySignature is key's cryptographic (HMAC) signature.
+	apikeySignature = 16
+	// apikeyLength is the length of the key in bytes.
+	apikeyLength = apikeyVersion + apikeyAppID + apikeySequence + apikeyWho + apikeySignature
+)
+
+// Client signature validation
+//
+//	key: client's secret key
+//
+// Returns application id, key type.
+func checkAPIKey(apikey string) (isValid, isRoot bool) {
+	if declen := base64.URLEncoding.DecodedLen(len(apikey)); declen != apikeyLength {
+		return
+	}
+
+	data, err := base64.URLEncoding.DecodeString(apikey)
+	if err != nil {
+		logs.Warn.Println("failed to decode.base64 appid ", err)
+		return
+	}
+	if data[0] != 1 {
+		logs.Warn.Println("unknown appid signature algorithm ", data[0])
+		return
+	}
+
+	hasher := hmac.New(md5.New, globals.apiKeySalt)
+	hasher.Write(data[:apikeyVersion+apikeyAppID+apikeySequence+apikeyWho])
+	check := hasher.Sum(nil)
+	if !bytes.Equal(data[apikeyVersion+apikeyAppID+apikeySequence+apikeyWho:], check) {
+		logs.Warn.Println("invalid apikey signature")
+		return
+	}
+
+	isRoot = (data[apikeyVersion+apikeyAppID+apikeySequence] == 1)
+
+	isValid = true
+
+	return
+}

server/auth/anon/auth_anon.go

@@ -0,0 +1,99 @@
+// Package anon provides authentication without credentials. Most useful for customer support.
+// Anonymous authentication is used only at the account creation time.
+package anon
+
+import (
+	"encoding/json"
+	"errors"
+	"time"
+
+	"github.com/tinode/chat/server/auth"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+// authenticator is the singleton instance of the anonymous authorizer.
+type authenticator struct {
+	name string
+}
+
+// Init is a noop, always returns success.
+func (a *authenticator) Init(_ json.RawMessage, name string) error {
+	if name == "" {
+		return errors.New("auth_anonymous: authenticator name cannot be blank")
+	}
+
+	if a.name != "" {
+		return errors.New("auth_anonymous: already initialized as " + a.name + "; " + name)
+	}
+
+	a.name = name
+	return nil
+}
+
+// IsInitialized returns true if the handler is initialized.
+func (a *authenticator) IsInitialized() bool {
+	return a.name != ""
+}
+
+// AddRecord checks authLevel and assigns default LevelAnon. Otherwise it
+// just reports success.
+func (authenticator) AddRecord(rec *auth.Rec, secret []byte, remoteAddr string) (*auth.Rec, error) {
+	if rec.AuthLevel == auth.LevelNone {
+		rec.AuthLevel = auth.LevelAnon
+	}
+	rec.State = types.StateOK
+	return rec, nil
+}
+
+// UpdateRecord is a noop. Just report success.
+func (authenticator) UpdateRecord(rec *auth.Rec, secret []byte, remoteAddr string) (*auth.Rec, error) {
+	return rec, nil
+}
+
+// Authenticate is not supported. This authenticator is used only at account creation time.
+func (authenticator) Authenticate(secret []byte, remoteAddr string) (*auth.Rec, []byte, error) {
+	return nil, nil, types.ErrUnsupported
+}
+
+// AsTag is not supported, will produce an empty string.
+func (authenticator) AsTag(token string) string {
+	return ""
+}
+
+// IsUnique for a noop. Anonymous login does not use secret, any secret is fine.
+func (authenticator) IsUnique(secret []byte, remoteAddr string) (bool, error) {
+	return true, nil
+}
+
+// GenSecret always fails.
+func (authenticator) GenSecret(rec *auth.Rec) ([]byte, time.Time, error) {
+	return nil, time.Time{}, types.ErrUnsupported
+}
+
+// DelRecords is a noop which always succeeds.
+func (authenticator) DelRecords(uid types.Uid) error {
+	return nil
+}
+
+// RestrictedTags returns tag namespaces restricted by this authenticator (none for anonymous).
+func (authenticator) RestrictedTags() ([]string, error) {
+	return nil, nil
+}
+
+// GetResetParams returns authenticator parameters passed to password reset handler
+// (none for anonymous).
+func (authenticator) GetResetParams(uid types.Uid) (map[string]any, error) {
+	return nil, nil
+}
+
+const realName = "anonymous"
+
+// GetRealName returns the hardcoded name of the authenticator.
+func (authenticator) GetRealName() string {
+	return realName
+}
+
+func init() {
+	store.RegisterAuthScheme(realName, &authenticator{})
+}

server/concurrency/goroutinepool.go

@@ -0,0 +1,56 @@
+// Package concurrency is a very simple implementation of a mutex with channels.
+// Provides TryLock functionality absent in Go's regular sync.Mutex.
+// See https://github.com/golang/go/issues/6123 for details.
+package concurrency
+
+// Task represents a work task to be run on the specified thread pool.
+type Task func()
+
+// GoRoutinePool is a pull of Go routines with associated locking mechanism.
+type GoRoutinePool struct {
+	// Work queue.
+	work chan Task
+	// Counter to control the number of already allocated/running goroutines.
+	sem chan struct{}
+	// Exit knob.
+	stop chan struct{}
+}
+
+// NewGoRoutinePool allocates a new thread pool with `numWorkers` goroutines.
+func NewGoRoutinePool(numWorkers int) *GoRoutinePool {
+	return &GoRoutinePool{
+		work: make(chan Task),
+		sem:  make(chan struct{}, numWorkers),
+		stop: make(chan struct{}, numWorkers),
+	}
+}
+
+// Schedule enqueus a closure to run on the GoRoutinePool's goroutines.
+func (p *GoRoutinePool) Schedule(task Task) {
+	select {
+	case p.work <- task:
+	case p.sem <- struct{}{}:
+		go p.worker(task)
+	}
+}
+
+// Stop sends a stop signal to all running goroutines.
+func (p *GoRoutinePool) Stop() {
+	numWorkers := cap(p.sem)
+	for range numWorkers {
+		p.stop <- struct{}{}
+	}
+}
+
+// Thread pool worker goroutine.
+func (p *GoRoutinePool) worker(task Task) {
+	defer func() { <-p.sem }()
+	for {
+		task()
+		select {
+		case task = <-p.work:
+		case <-p.stop:
+			return
+		}
+	}
+}

server/concurrency/simplemutex.go

@@ -0,0 +1,30 @@
+package concurrency
+
+// SimpleMutex is a channel used for locking.
+type SimpleMutex chan struct{}
+
+// NewSimpleMutex creates and returns a new SimpleMutex object.
+func NewSimpleMutex() SimpleMutex {
+	return make(SimpleMutex, 1)
+}
+
+// Lock acquires a lock on the mutex.
+func (s SimpleMutex) Lock() {
+	s <- struct{}{}
+}
+
+// TryLock attempts to acquire a lock on the mutex.
+// Returns true if the lock has been acquired, false otherwise.
+func (s SimpleMutex) TryLock() bool {
+	select {
+	case s <- struct{}{}:
+		return true
+	default:
+		return false
+	}
+}
+
+// Unlock releases the mutex.
+func (s SimpleMutex) Unlock() {
+	<-s
+}

server/db/mongodb/blank.go

@@ -0,0 +1,7 @@
+//go:build !mongodb
+// +build !mongodb
+
+// This file is needed for conditional compilation. It's used when
+// the build tag 'mongodb' is not defined. Otherwise the adapter.go
+// is compiled.
+package mongodb

server/db/mysql/blank.go

@@ -0,0 +1,8 @@
+//go:build !mysql
+// +build !mysql
+
+// This file is needed for conditional compilation. It's used when
+// the build tag 'mysql' is not defined. Otherwise the adapter.go
+// is compiled.
+
+package mysql

server/db/postgres/blank.go

@@ -0,0 +1,8 @@
+//go:build !postgres
+// +build !postgres
+
+// This file is needed for conditional compilation. It's used when
+// the build tag 'postgres' is not defined. Otherwise the adapter.go
+// is compiled.
+
+package postgres

server/db/rethinkdb/blank.go

@@ -0,0 +1,8 @@
+//go:build !rethinkdb
+// +build !rethinkdb
+
+// This file is needed for conditional compilation. It's used when
+// the build tag 'rethinkdb' is not defined. Otherwise the adapter.go
+// is compiled.
+
+package rethinkdb

server/drafty/grapheme.go

@@ -0,0 +1,77 @@
+package drafty
+
+import (
+	"github.com/rivo/uniseg"
+)
+
+// graphemes is a container holding lengths of grapheme clusters in a string.
+type graphemes struct {
+	// The original string.
+	original string
+
+	// Sizes of grapheme clusters within the original string.
+	sizes []byte
+}
+
+// prepareGraphemes returns a parsed grapheme cluster container by splitting the string into grapheme clusters
+// and saving their lengths.
+func prepareGraphemes(str string) *graphemes {
+	// Split the string into grapheme clusters and save the size of each cluster.
+	sizes := make([]byte, 0, len(str))
+	for state, remaining, cluster := -1, str, ""; len(remaining) > 0; {
+		cluster, remaining, _, state = uniseg.StepString(remaining, state)
+		sizes = append(sizes, byte(len(cluster)))
+	}
+
+	return &graphemes{
+		original: str,
+		sizes:    sizes,
+	}
+}
+
+// length returns the number of grapheme clusters in the original string.
+func (g *graphemes) length() int {
+	if g == nil {
+		return 0
+	}
+	return len(g.sizes)
+}
+
+// string returns the original string from which the grapheme cluster container was created.
+func (g *graphemes) string() string {
+	if g == nil {
+		return ""
+	}
+	return g.original
+}
+
+// slice returns a new grapheme cluster container with grapheme clusters from 'start' to 'end'.
+func (g *graphemes) slice(start, end int) *graphemes {
+
+	// Convert grapheme offsets to string offsets.
+	s := 0
+	for i := range start {
+		s += int(g.sizes[i])
+	}
+	e := s
+	for i := start; i < end; i++ {
+		e += int(g.sizes[i])
+	}
+
+	return &graphemes{
+		original: g.original[s:e],
+		sizes:    g.sizes[start:end],
+	}
+}
+
+// append appends 'other' grapheme cluster container to 'g' container and returns g.
+// If g is nil, the 'other' is returned.
+func (g *graphemes) append(other *graphemes) *graphemes {
+	if g == nil {
+		return other
+	}
+
+	g.original += other.original
+	g.sizes = append(g.sizes, other.sizes...)
+	return g
+}

server/http_pprof.go

@@ -0,0 +1,53 @@
+// Debug tooling. Dumps named profile in response to HTTP request at
+// 		http(s)://<host-name>/<configured-path>/<profile-name>
+// See godoc for the list of possible profile names: https://golang.org/pkg/runtime/pprof/#Profile
+
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"path"
+	"runtime/pprof"
+	"strings"
+
+	"github.com/tinode/chat/server/logs"
+)
+
+var pprofHttpRoot string
+
+// Expose debug profiling at the given URL path.
+func servePprof(mux *http.ServeMux, serveAt string) {
+	if serveAt == "" || serveAt == "-" {
+		return
+	}
+
+	pprofHttpRoot = path.Clean("/"+serveAt) + "/"
+	mux.HandleFunc(pprofHttpRoot, profileHandler)
+
+	logs.Info.Printf("pprof: profiling info exposed at '%s'", pprofHttpRoot)
+}
+
+func profileHandler(wrt http.ResponseWriter, req *http.Request) {
+	wrt.Header().Set("X-Content-Type-Options", "nosniff")
+	wrt.Header().Set("Content-Type", "text/plain; charset=utf-8")
+
+	profileName := strings.TrimPrefix(req.URL.Path, pprofHttpRoot)
+
+	profile := pprof.Lookup(profileName)
+	if profile == nil {
+		servePprofError(wrt, http.StatusNotFound, "Unknown profile '"+profileName+"'")
+		return
+	}
+
+	// Respond with the requested profile.
+	profile.WriteTo(wrt, 2)
+}
+
+func servePprofError(wrt http.ResponseWriter, status int, txt string) {
+	wrt.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	wrt.Header().Set("X-Go-Pprof", "1")
+	wrt.Header().Del("Content-Disposition")
+	wrt.WriteHeader(status)
+	fmt.Fprintln(wrt, txt)
+}

server/logs/logs.go

@@ -0,0 +1,55 @@
+// Package logs exposes info, warning and error loggers.
+package logs
+
+import (
+	"io"
+	"log"
+	"strings"
+)
+
+var (
+	// Info is a logger at the 'info' logging level.
+	Info *log.Logger
+	// Warn is a logger at the 'warning' logging level.
+	Warn *log.Logger
+	// Err is a logger at the 'error' logging level.
+	Err *log.Logger
+)
+
+func parseFlags(logFlags string) int {
+	flags := 0
+	for _, v := range strings.Split(logFlags, ",") {
+		switch {
+		case v == "date":
+			flags |= log.Ldate
+		case v == "time":
+			flags |= log.Ltime
+		case v == "microseconds":
+			flags |= log.Lmicroseconds
+		case v == "longfile":
+			flags |= log.Llongfile
+		case v == "shortfile":
+			flags |= log.Lshortfile
+		case v == "UTC":
+			flags |= log.LUTC
+		case v == "msgprefix":
+			flags |= log.Lmsgprefix
+		case v == "stdFlags":
+			flags |= log.LstdFlags
+		default:
+			log.Fatalln("Invalid log flags string: ", logFlags)
+		}
+	}
+	if flags == 0 {
+		flags = log.LstdFlags
+	}
+	return flags
+}
+
+// Init initializes info, warning and error loggers given the flags and the output.
+func Init(output io.Writer, logFlags string) {
+	flags := parseFlags(logFlags)
+	Info = log.New(output, "I", flags)
+	Warn = log.New(output, "W", flags)
+	Err = log.New(output, "E", flags)
+}

server/push/stdout/push_stdout.go

@@ -0,0 +1,98 @@
+// Package stdout is a sample implementation of a push plugin.
+// If enabled, it writes every notification to stdout.
+package stdout
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/tinode/chat/server/push"
+)
+
+var handler stdoutPush
+
+// How much to buffer the input channel.
+const defaultBuffer = 32
+
+type stdoutPush struct {
+	initialized bool
+	input       chan *push.Receipt
+	channel     chan *push.ChannelReq
+	stop        chan bool
+}
+
+type configType struct {
+	Enabled bool `json:"enabled"`
+	Buffer  int  `json:"buffer"`
+}
+
+// Init initializes the handler
+func (stdoutPush) Init(jsonconf json.RawMessage) (bool, error) {
+
+	// Check if the handler is already initialized
+	if handler.initialized {
+		return false, errors.New("already initialized")
+	}
+
+	var config configType
+	if err := json.Unmarshal([]byte(jsonconf), &config); err != nil {
+		return false, errors.New("failed to parse config: " + err.Error())
+	}
+
+	handler.initialized = true
+
+	if !config.Enabled {
+		return false, nil
+	}
+
+	if config.Buffer <= 0 {
+		config.Buffer = defaultBuffer
+	}
+
+	handler.input = make(chan *push.Receipt, config.Buffer)
+	handler.channel = make(chan *push.ChannelReq, config.Buffer)
+	handler.stop = make(chan bool, 1)
+
+	go func() {
+		for {
+			select {
+			case msg := <-handler.input:
+				fmt.Fprintln(os.Stdout, msg)
+			case msg := <-handler.channel:
+				fmt.Fprintln(os.Stdout, msg)
+			case <-handler.stop:
+				return
+			}
+		}
+	}()
+
+	return true, nil
+}
+
+// IsReady checks if the handler is initialized.
+func (stdoutPush) IsReady() bool {
+	return handler.input != nil
+}
+
+// Push returns a channel that the server will use to send messages to.
+// If the adapter blocks, the message will be dropped.
+func (stdoutPush) Push() chan<- *push.Receipt {
+	return handler.input
+}
+
+// Channel returns a channel that caller can use to subscribe/unsubscribe devices to channels (FCM topics).
+// If the adapter blocks, the message will be dropped.
+func (stdoutPush) Channel() chan<- *push.ChannelReq {
+	return handler.channel
+}
+
+// Stop terminates the handler's worker and stops sending pushes.
+func (stdoutPush) Stop() {
+	handler.stop <- true
+}
+
+func init() {
+	push.Register("stdout", &handler)
+}

server/ringhash/ringhash.go

@@ -0,0 +1,134 @@
+// Package ringhash implementats a consistent ring hash:
+// https://en.wikipedia.org/wiki/Consistent_hashing
+package ringhash
+
+import (
+	"encoding/ascii85"
+	"hash/crc32"
+	"hash/fnv"
+	"sort"
+	"strconv"
+
+	"github.com/tinode/chat/server/logs"
+)
+
+// Hash is a signature of a hash function used by the package.
+type Hash func(data []byte) uint32
+
+type elem struct {
+	key  string
+	hash uint32
+}
+
+type sortable []elem
+
+func (k sortable) Len() int      { return len(k) }
+func (k sortable) Swap(i, j int) { k[i], k[j] = k[j], k[i] }
+func (k sortable) Less(i, j int) bool {
+	// Weak hash function may cause collisions.
+	if k[i].hash < k[j].hash {
+		return true
+	}
+	if k[i].hash == k[j].hash {
+		return k[i].key < k[j].key
+	}
+	return false
+}
+
+// Ring is the definition of the ringhash.
+type Ring struct {
+	keys []elem // Sorted list of keys.
+
+	signature string
+	replicas  int
+	hashfunc  Hash
+}
+
+// New initializes an empty ringhash with the given number of replicas and a hash function.
+// If the hash function is nil, crc32.NewIEEE() is used.
+func New(replicas int, fn Hash) *Ring {
+	ring := &Ring{
+		replicas: replicas,
+		hashfunc: fn,
+	}
+	if ring.hashfunc == nil {
+		ring.hashfunc = func(data []byte) uint32 {
+			hash := crc32.NewIEEE()
+			hash.Write(data)
+			return hash.Sum32()
+		}
+	}
+	return ring
+}
+
+// Len returns the number of keys in the ring.
+func (ring *Ring) Len() int {
+	return len(ring.keys)
+}
+
+// Add adds keys to the ring.
+func (ring *Ring) Add(keys ...string) {
+	for _, key := range keys {
+		for i := range ring.replicas {
+			ring.keys = append(ring.keys, elem{
+				hash: ring.hashfunc([]byte(strconv.Itoa(i) + key)),
+				key:  key})
+		}
+	}
+	sort.Sort(sortable(ring.keys))
+
+	// Calculate signature
+	hash := fnv.New128a()
+	b := make([]byte, 4)
+	for _, key := range ring.keys {
+		b[0] = byte(key.hash)
+		b[1] = byte(key.hash >> 8)
+		b[2] = byte(key.hash >> 16)
+		b[3] = byte(key.hash >> 24)
+		hash.Write(b)
+		hash.Write([]byte(key.key))
+	}
+
+	b = []byte{}
+	b = hash.Sum(b)
+	dst := make([]byte, ascii85.MaxEncodedLen(len(b)))
+	ascii85.Encode(dst, b)
+	ring.signature = string(dst)
+}
+
+// Get returns the closest item in the ring to the provided key.
+func (ring *Ring) Get(key string) string {
+
+	if ring.Len() == 0 {
+		return ""
+	}
+
+	hash := ring.hashfunc([]byte(key))
+
+	// Binary search for appropriate replica.
+	idx := sort.Search(len(ring.keys), func(i int) bool {
+		el := ring.keys[i]
+		return (el.hash > hash) || (el.hash == hash && el.key >= key)
+	})
+
+	// Means we have cycled back to the first replica.
+	if idx == len(ring.keys) {
+		idx = 0
+	}
+
+	return ring.keys[idx].key
+}
+
+// Signature returns the ring's hash signature. Two identical ringhashes
+// will have the same signature. Two hashes with different
+// number of keys or replicas or hash functions will have different
+// signatures.
+func (ring *Ring) Signature() string {
+	return ring.signature
+}
+
+func (ring *Ring) dump() {
+	for _, e := range ring.keys {
+		logs.Info.Printf("key: '%s', hash=%d", e.key, e.hash)
+	}
+}

server/stats.go

@@ -0,0 +1,167 @@
+// Logic related to expvar handling: reporting live stats such as
+// session and topic counts, memory usage etc.
+// The stats updates happen in a separate go routine to avoid
+// locking on main logic routines.
+
+package main
+
+import (
+	"encoding/json"
+	"expvar"
+	"net/http"
+	"runtime"
+	"sort"
+	"time"
+
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store"
+)
+
+// A simple implementation of histogram expvar.Var.
+// `Bounds` specifies the histogram buckets as follows (length = len(bounds)):
+//
+//	(-inf, Bounds[i]) for i = 0
+//	[Bounds[i-1], Bounds[i]) for 0 < i < length
+//	[Bounds[i-1], +inf) for i = length
+type histogram struct {
+	Count          int64     `json:"count"`
+	Sum            float64   `json:"sum"`
+	CountPerBucket []int64   `json:"count_per_bucket"`
+	Bounds         []float64 `json:"bounds"`
+}
+
+func (h *histogram) addSample(v float64) {
+	h.Count++
+	h.Sum += v
+	idx := sort.SearchFloat64s(h.Bounds, v)
+	h.CountPerBucket[idx]++
+}
+
+func (h *histogram) String() string {
+	if r, err := json.Marshal(h); err == nil {
+		return string(r)
+	}
+	return ""
+}
+
+type varUpdate struct {
+	// Name of the variable to update
+	varname string
+	// Value to publish (int, float, etc.)
+	value any
+	// Treat the count as an increment as opposite to the final value.
+	inc bool
+}
+
+// Initialize stats reporting through expvar.
+func statsInit(mux *http.ServeMux, path string) {
+	if path == "" || path == "-" {
+		return
+	}
+
+	mux.Handle(path, expvar.Handler())
+	globals.statsUpdate = make(chan *varUpdate, 1024)
+
+	start := time.Now()
+	expvar.Publish("Uptime", expvar.Func(func() any {
+		return time.Since(start).Seconds()
+	}))
+	expvar.Publish("NumGoroutines", expvar.Func(func() any {
+		return runtime.NumGoroutine()
+	}))
+
+	go statsUpdater()
+
+	logs.Info.Printf("stats: variables exposed at '%s'", path)
+}
+
+func statsRegisterDbStats() {
+	if f := store.Store.DbStats(); f != nil {
+		expvar.Publish("DbStats", expvar.Func(f))
+	}
+}
+
+// Register integer variable. Don't check for initialization.
+func statsRegisterInt(name string) {
+	expvar.Publish(name, new(expvar.Int))
+}
+
+// Register histogram variable. `bounds` specifies histogram buckets/bins
+// (see comment next to the `histogram` struct definition).
+func statsRegisterHistogram(name string, bounds []float64) {
+	numBuckets := len(bounds) + 1
+	expvar.Publish(name, &histogram{
+		CountPerBucket: make([]int64, numBuckets),
+		Bounds:         bounds,
+	})
+}
+
+// Async publish int variable.
+func statsSet(name string, val int64) {
+	if globals.statsUpdate != nil {
+		select {
+		case globals.statsUpdate <- &varUpdate{name, val, false}:
+		default:
+		}
+	}
+}
+
+// Async publish an increment (decrement) to int variable.
+func statsInc(name string, val int) {
+	if globals.statsUpdate != nil {
+		select {
+		case globals.statsUpdate <- &varUpdate{name, int64(val), true}:
+		default:
+		}
+	}
+}
+
+// Async publish a value (add a sample) to a histogram variable.
+func statsAddHistSample(name string, val float64) {
+	if globals.statsUpdate != nil {
+		select {
+		case globals.statsUpdate <- &varUpdate{varname: name, value: val}:
+		default:
+		}
+	}
+}
+
+// Stop publishing stats.
+func statsShutdown() {
+	if globals.statsUpdate != nil {
+		globals.statsUpdate <- nil
+	}
+}
+
+// The go routine which actually publishes stats updates.
+func statsUpdater() {
+	for upd := range globals.statsUpdate {
+		if upd == nil {
+			globals.statsUpdate = nil
+			// Dont' care to close the channel.
+			break
+		}
+
+		// Handle var update
+		if ev := expvar.Get(upd.varname); ev != nil {
+			switch v := ev.(type) {
+			case *expvar.Int:
+				count := upd.value.(int64)
+				if upd.inc {
+					v.Add(count)
+				} else {
+					v.Set(count)
+				}
+			case *histogram:
+				val := upd.value.(float64)
+				v.addSample(val)
+			default:
+				logs.Err.Panicf("stats: unsupported expvar type %T", ev)
+			}
+		} else {
+			panic("stats: update to unknown variable " + upd.varname)
+		}
+	}
+
+	logs.Info.Println("stats: shutdown")
+}

server/store/types/uidgen.go

@@ -0,0 +1,105 @@
+package types
+
+import (
+	"encoding/base64"
+	"encoding/binary"
+	"errors"
+
+	sf "github.com/tinode/snowflake"
+	"golang.org/x/crypto/xtea"
+)
+
+// UidGenerator holds snowflake and encryption paramenets.
+// RethinkDB generates UUIDs as primary keys. Using snowflake-generated uint64 instead.
+type UidGenerator struct {
+	seq    *sf.SnowFlake
+	cipher *xtea.Cipher
+}
+
+var ErrUninitialized = errors.New("uninitialized")
+
+// Init initialises the Uid generator
+func (ug *UidGenerator) Init(workerID uint, key []byte) error {
+	var err error
+
+	if ug.seq == nil {
+		ug.seq, err = sf.NewSnowFlake(uint32(workerID))
+	}
+	if err == nil && ug.cipher == nil {
+		ug.cipher, err = xtea.NewCipher(key)
+	}
+
+	return err
+}
+
+// Get generates a unique weakly-encryped random-looking ID.
+// The Uid is a unit64 with the highest bit possibly set which makes it
+// incompatible with go's pre-1.9 sql package.
+func (ug *UidGenerator) Get() Uid {
+	buf, err := getIDBuffer(ug)
+	if err != nil {
+		return ZeroUid
+	}
+	return Uid(binary.LittleEndian.Uint64(buf))
+}
+
+// GetStr generates the same unique ID as Get then returns it as
+// base64-encoded string. Slightly more efficient than calling Get()
+// then base64-encoding the result.
+func (ug *UidGenerator) GetStr() string {
+	buf, err := getIDBuffer(ug)
+	if err != nil {
+		return ""
+	}
+	return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(buf)
+}
+
+// getIdBuffer returns a byte array holding the Uid bytes
+func getIDBuffer(ug *UidGenerator) ([]byte, error) {
+	if ug == nil || ug.seq == nil {
+		return nil, ErrUninitialized
+	}
+
+	var id uint64
+	var err error
+	if id, err = ug.seq.Next(); err != nil {
+		return nil, err
+	}
+
+	src := make([]byte, 8)
+	dst := make([]byte, 8)
+	binary.LittleEndian.PutUint64(src, id)
+	ug.cipher.Encrypt(dst, src)
+
+	return dst, nil
+}
+
+// DecodeUid takes an encrypted Uid and decrypts it into a non-negative int64.
+// This is needed for go/sql compatibility where uint64 with high bit
+// set is unsupported and possibly for other uses such as MySQL's recommendation
+// for sequential primary keys.
+func (ug *UidGenerator) DecodeUid(uid Uid) int64 {
+	if ug == nil || ug.seq == nil {
+		return 0
+	}
+	src := make([]byte, 8)
+	dst := make([]byte, 8)
+	binary.LittleEndian.PutUint64(src, uint64(uid))
+	ug.cipher.Decrypt(dst, src)
+	return int64(binary.LittleEndian.Uint64(dst))
+}
+
+// EncodeInt64 takes a positive int64 and encrypts it into a Uid.
+// This is needed for go/sql compatibility where uint64 with high bit
+// set is unsupported  and possibly for other uses such as MySQL's recommendation
+// for sequential primary keys.
+func (ug *UidGenerator) EncodeInt64(val int64) Uid {
+	if ug == nil || ug.seq == nil {
+		return ZeroUid
+	}
+	src := make([]byte, 8)
+	dst := make([]byte, 8)
+	binary.LittleEndian.PutUint64(src, uint64(val))
+	ug.cipher.Encrypt(dst, src)
+	return Uid(binary.LittleEndian.Uint64(dst))
+}