A Golang implementation of the Yjs algorithms, designed to serve as a robust backend server for multi-terminal document collaboration. This implementation enhances real-time collaboration experiences across diverse user scenarios by efficiently merging updates from various terminals, extracting differential data, and supporting data archiving.
In the future, we plan to develop a complete y-server service that synchronizes data with client terminals via WebSocket. Stay tuned for updates!
Test cases are implemented in compatibility_test.go , focusing on validating cross-version and cross-language compatibility with Yjs.
Note: Encoder/decoder v2 support is pending development.
Compatibility testing passed.
package y_crdt
import (
"bytes"
"encoding/json"
"testing"
"github.com/bytedance/mockey"
)
func TestTextInsertDelete(t *testing.T) {
// Generated via:
// ```js
// const doc = new Y.Doc()
// const ytext = doc.getText('type')
// doc..transact_mut()(function () {
// ytext.insert(0, 'def')
// ytext.insert(0, 'abc')
// ytext.insert(6, 'ghi')
// ytext.delete(2, 5)
// })
// const update = Y.encodeStateAsUpdate(doc)
// ytext.toString() // => 'abhi'
// ```
//
// This way we confirm that we can decode and apply:
// 1. blocks without left/right origin consisting of multiple characters
// 2. blocks with left/right origin consisting of multiple characters
// 3. delete sets
// construct doc by golang and check to see if the result is the same as the expected.
doc := NewDoc("guid", false, nil, nil, false)
ytext := doc.GetText("type")
doc.Transact(func(trans *Transaction) {
ytext.Insert(0, "def", nil)
ytext.Insert(0, "abc", nil)
ytext.Insert(6, "ghi", nil)
ytext.Delete(2, 5)
}, nil)
if ytext.ToString() != "abhi" {
t.Error("expected abhi, got ", ytext.ToString())
}
t.Logf("construct by golang, ytext is %s", ytext.ToString())
// the payload was generated by javascript.
var payload = []byte{
1, 5, 152, 234, 173, 126, 0, 1, 1, 4, 116, 121, 112, 101, 3, 68, 152, 234, 173, 126, 0, 2,
97, 98, 193, 152, 234, 173, 126, 4, 152, 234, 173, 126, 0, 1, 129, 152, 234, 173, 126, 2,
1, 132, 152, 234, 173, 126, 6, 2, 104, 105, 1, 152, 234, 173, 126, 2, 0, 3, 5, 2,
}
// apply the update and check to see if the result is the same as the expected.
doc = NewDoc("guid", false, nil, nil, false)
doc.Transact(func(trans *Transaction) {
ApplyUpdate(doc, payload, nil)
}, nil)
ytext = doc.GetText("type")
if ytext.ToString() != "abhi" {
t.Errorf("expected abhi, got %s", ytext.ToString())
}
t.Logf("after apply update, ytext is %s", ytext.ToString())
}
func TestMapSet(t *testing.T) {
// Generated via:
// ```js
// const doc = new Y.Doc()
// const x = doc.getMap('test')
// x.set('k1', 'v1')
// x.set('k2', 'v2')
// const payload_v1 = Y.encodeStateAsUpdate(doc)
// console.log(payload_v1);
// const payload_v2 = Y.encodeStateAsUpdateV2(doc)
// console.log(payload_v2);
// ```
mocker := mockey.Mock(GenerateNewClientID).Return(440166001).Build()
// construct doc by golang and check to see if the result is the same as the expected.
doc := NewDoc("guid", false, nil, nil, false)
x := doc.GetMap("test").(*YMap)
doc.Transact(func(trans *Transaction) {
x.Set("k1", "v1")
x.Set("k2", "v2")
}, nil)
content, err := json.Marshal(x.ToJson())
if err != nil {
t.Errorf("marshal x to json, err is %v", err)
}
t.Logf("construct by golang, x is %s", content)
m := make(map[string]string)
err = json.Unmarshal(content, &m)
if err != nil || len(m) != 2 || m["k1"] != "v1" || m["k2"] != "v2" {
t.Errorf("expected {\"k1\":\"v1\",\"k2\":\"v2\"}, got %s", content)
}
// the payload was generated by javascript.
var payload = []byte{
1, 2, 241, 204, 241, 209, 1, 0, 40, 1, 4, 116, 101, 115, 116, 2, 107, 49, 1, 119, 2, 118,
49, 40, 1, 4, 116, 101, 115, 116, 2, 107, 50, 1, 119, 2, 118, 50, 0,
}
// encode doc(geneareted by golang) and compare with payload(generated by javascript).
update := EncodeStateAsUpdate(doc, nil)
t.Logf("update is %v", update)
if !bytes.Equal(update, payload) {
t.Errorf("expect update:%v got update:%v", payload, update)
}
// apply the update(v1) and check to see if the result is the same as the expected.
mocker.UnPatch()
doc = NewDoc("guid", false, nil, nil, false)
doc.Transact(func(trans *Transaction) {
ApplyUpdate(doc, payload, nil)
}, nil)
content, err = json.Marshal(doc.GetMap("test").ToJson())
if err != nil {
t.Errorf("marshal doc.GetMap(\"test\") to json, err is %v", err)
}
t.Logf("after apply update, x is %s", content)
m = make(map[string]string)
err = json.Unmarshal(content, &m)
if err != nil || len(m) != 2 || m["k1"] != "v1" || m["k2"] != "v2" {
t.Errorf("expected {\"k1\":\"v1\",\"k2\":\"v2\"}, got %s", content)
}
// decoder v2 not support yet.
}
func TestArrayInsert(t *testing.T) {
// Generated via:
// ```js
// const doc = new Y.Doc()
// const x = doc.getArray('test')
// x.push(['a']);
// x.push(['b']);
// const payload_v1 = Y.encodeStateAsUpdate(doc)
// console.log(payload_v1);
// const payload_v2 = Y.encodeStateAsUpdateV2(doc)
// console.log(payload_v2);
// ```
mocker := mockey.Mock(GenerateNewClientID).Return(2525665872).Build()
// construct doc by golang and check to see if the result is the same as the expected.
doc := NewDoc("guid", false, nil, nil, false)
x := doc.GetArray("test")
x.Push([]any{"a"})
x.Push([]any{"b"})
content, err := json.Marshal(x.ToJson())
t.Logf("construct by golang, x is %s, err is %v", content, err)
if !bytes.Equal(content, []byte("[\"a\",\"b\"]")) {
t.Errorf("expected [\"a\",\"b\"], got %s", content)
}
// the payload was generated by javascript.
var payload = []byte{
1, 1,
2E87
208, 180, 170, 180, 9, 0, 8, 1, 4, 116, 101, 115, 116, 2, 119, 1, 97, 119, 1, 98, 0,
}
// encode doc(geneareted by golang) and compare with payload(generated by javascript).
update := EncodeStateAsUpdate(doc, nil)
t.Logf("update is %v", update)
if !bytes.Equal(update, payload) {
t.Errorf("expect update:%v got update:%v", payload, update)
}
// apply the update(v1) and check to see if the result is the same as the expected.
mocker.UnPatch()
doc = NewDoc("new doc", false, nil, nil, false)
ApplyUpdate(doc, payload, nil)
content, err = json.Marshal(doc.GetArray("test").ToJson())
t.Logf("after apply update, x is %s, err is %v", content, err)
if !bytes.Equal(content, []byte("[\"a\",\"b\"]")) {
t.Errorf("expected [\"a\",\"b\"], got %s", content)
}
}
func TestXmlFragmentInsert(t *testing.T) {
// Generated via:
// ```js
// const ydoc = new Y.Doc()
// const yxmlFragment = ydoc.getXmlFragment('fragment-name')
// const yxmlNested = new Y.XmlFragment('fragment-name')
// const yxmlText = new Y.XmlText()
// yxmlFragment.insert(0, [yxmlText])
// yxmlFragment.firstChild === yxmlText
// yxmlFragment.insertAfter(yxmlText, [new Y.XmlElement('node-name')])
// const payload_v1 = Y.encodeStateAsUpdate(ydoc)
// console.log(payload_v1);
// const payload_v2 = Y.encodeStateAsUpdateV2(ydoc)
// console.log(payload_v2);
// ```
mockey.Mock(GenerateNewClientID).Return(2459881872).Build()
// construct doc by golang and check to see if the result is the same as the expected.
doc := NewDoc("guid", false, nil, nil, false)
yxmlFragment := doc.GetXmlFragment("fragment-name").(*YXmlFragment)
yxmlText := NewYXmlText()
yxmlFragment.Insert(0, ArrayAny{yxmlText})
if yxmlFragment.GetFirstChild().(*YXmlText) != yxmlText {
t.Errorf("expected yxmlFragment.GetFirstChild() is yxmlText, got %v", yxmlFragment.GetFirstChild())
}
yxmlFragment.InsertAfter(yxmlText, ArrayAny{NewYXmlElement("node-name")})
update := EncodeStateAsUpdate(doc, nil)
var payload = []byte{
1, 2, 144, 163, 251, 148, 9, 0, 7, 1, 13, 102, 114, 97, 103, 109, 101, 110, 116, 45, 110,
97, 109, 101, 6, 135, 144, 163, 251, 148, 9, 0, 3, 9, 110, 111, 100, 101, 45, 110, 97, 109,
101, 0,
}
if !bytes.Equal(update, payload) {
t.Errorf("expected update:%v got update:%v", payload, update)
}
}
func TestStateVector(t *testing.T) {
// Generated via:
// ```js
// const a = new Y.Doc()
// const ta = a.getText('test')
// ta.insert(0, 'abc')
// const b = new Y.Doc()
// const tb = b.getText('test')
// tb.insert(0, 'de')
// Y.applyUpdate(a, Y.encodeStateAsUpdate(b))
// console.log(Y.encodeStateVector(a))
// ```
var payload = []byte{2, 178, 219, 218, 44, 3, 190, 212, 225, 6, 2}
sv := DecodeStateVector(payload)
expected := map[Number]Number{
14182974: 2,
93760946: 3,
}
for k, v := range expected {
if sv[k] != v {
t.Errorf("expected %v, got %v", v, sv[k])
}
}
serialized := EncodeStateVector(nil, sv, NewUpdateEncoderV1())
if !bytes.Equal(serialized, payload) {
t.Errorf("expected: %v, got: %v", payload, serialized)
}
}
- WriteByte: Write
uint8to buffer. - WriteVarUint: Variable-length
uint64encoding. - WriteVarInt: Variable-length
intencoding with sign handling. - WriteFloat32/64: Big-endian float encoding.
- WriteInt64: Big-endian
int64encoding.
- WriteString: Write length + string data.
- WriteObject: Write key-value count, then key-value pairs.
- WriteArray: Write element count, then elements.
- WriteVarUint8Array/WriteUint8Array: Write byte arrays with/without length prefix.
- WriteAny: Type-specific encoding with flag bytes.
- readVarUint: Read variable-length
uint64. - ReadUint8: Read
uint8. - ReadVarInt: Restore
intfrom variable-length bytes. - ReadFloat32/64: Parse big-endian floats.
- ReadBigInt64: Parse big-endian
int64.
- ReadString: Read length, then string data.
- ReadObject: Read count, then key-value pairs.
- ReadArray: Read count, then elements.
- ReadVarUint8Array: Read length, then byte array.
- ReadAny: Decode based on type flag byte.
Encoding and decoding are symmetric, using variable-length and big-endian formats for efficiency.
- YMap: A key-value store with efficient updates.
- YMapItem: Represents a key-value pair in the map.
- YMapItemMap: Maps keys to YMapItem pointers.
- YArray: A dynamic array with efficient updates.
- YArrayItem: Represents an element in the array.
- YText: A text document with efficient updates.
- YTextItem: Represents a character in the text.
- YTextItemMap: Maps positions to YTextItem pointers.
- YXmlFragment: A fragment of an XML document.
- YXmlFragmentItem: Represents an XML node.
- YXmlFragmentItemMap: Maps positions to YXmlFragmentItem pointers.