如何将json压缩文件解组到结构中

如何将json压缩文件解组到结构中

问题描述:

I am writing a trie DS, json gzipped into a file trieSample.json.gz, and reading it back into the struct. Strangely, the unmarshal succeeds but the struct is not populated.

I have tried both json.Unmarshal and json.Decoder to no avail. Need help in finding what I am missing here. There is no error thrown while reading, just that the struct doesn't have any keys. If I try normal json marshal -> Write to file and Read from file -> Unmarshal, it works properly.

var charSet = "0123456789bcdefghjkmnopqrstuvwxyz"
const logTagSlice = "trie.log"

type trieSlice struct {
    Children []*tNode          `json:"c"`
    Charset  map[int32]int8    `json:"l"` // Charset is the legend of what charset is used to create the keys and how to position them in trie array
    logger   loggingapi.Logger `json:"-"`
    capacity int               `json:"-"` // capacity is the slice capacity to have enough to hold all the characters in the charset
}

type tNode struct {
    Children []*tNode `json:"c"`           // children represents the next valid runes AFTER the current one
    IsLeaf   bool     `json:"e,omitempty"` // isLeaf represents if this node represents the end of a valid word
    Value    int16    `json:"v,omitempty"` // value holds the corresponding value for key value pair, where key is the whole tree of nodes starting from parent representing a valid word
}

// NewTrieSlice returns a Trie, charset represents how the children need to be positioned in the array
func NewTrieSlice(charset string, logger loggingapi.Logger) *trieSlice {
    m := map[int32]int8{}
    for index, r := range charset {
        m[r] = int8(index)
    }
    return &trieSlice{
        Charset:  m,
        Children: make([]*tNode, len(charset)),
        logger:   logger,
        capacity: len(charset),
    }
}

func newNode(capacity int) *tNode {
    return &tNode{
        Children: make([]*tNode, capacity),
    }
}

// getPosition gets the array index position that the rune should be put in
func (t *trieSlice) getPosition(r int32) (index int8, found bool) {
    if index, ok := t.Charset[r]; ok {
        return index, true
    }

    return -1, false
}

// Add ...
func (t *trieSlice) Add(key string, val int16) {
    if len(key) == 0 {
        t.logger.Info(logTagSlice, "trying to add empty key, return with no action")
        return
    }

    runes := []rune(key)
    prefix := runes[0]
    var child *tNode
    var pos int

    index, ok := t.getPosition(prefix)
    if !ok {
        t.logger.Info(logTagSlice, "key is not present in the charset %s,  cannot add to trieSlice", prefix)
        return
    }

    // trie node with same prefix doesnt exist
    if child = t.Children[index]; child == nil {
        child = newNode(len(t.Charset))
        t.Children[index] = child
    }

    pos = 1
    for pos <= len(runes) {
        // base condition
        if pos == len(key) {
            child.IsLeaf = true
            child.Value = val
            return
        }

        prefix := runes[pos]
        index, ok := t.getPosition(prefix)
        if !ok {
            t.logger.Info(logTagSlice, "key is not present in the charset %s,  cannot add to trieSlice", prefix)
            return
        }

        // repeat with child node if prefix is already present
        if newChild := child.Children[index]; newChild == nil {
            child.Children[index] = newNode(len(t.Charset))
            child = child.Children[index]
        } else {
            child = newChild
        }
        pos++
    }
}

// Test using gzip writer, reader
func TestSample(t *testing.T) {
    // Create trie and add a few keys
    trie := NewTrieSlice(charSet, loggingapi.NewStdOut())
    trie.Add("test", 10)
    trie.Add("test1", 20)
    trie.Add("test2", 30)
    trie.Add("test3", 40)
    trie.Add("test4", 50)

    // Write gzipped json to file
    var network bytes.Buffer
    b, err := json.Marshal(trie)
    if err != nil {
        fmt.Println("error in marshal ... ", err.Error())
        t.Fail()
    }
    w := gzip.NewWriter(&network)
    w.Write(b)
    ioutil.WriteFile("../resources/trieSample.json.gz", []byte(network.String()), 0644)
    w.Close()

    // Read gzipped json from file into struct
    trieUnmarshal := NewTrieSlice(charSet, loggingapi.NewStdOut())
    trieDecoder := NewTrieSlice(charSet, loggingapi.NewStdOut())

    // attempt via json Unmarshal
    file, err := os.Open("../resources/trieSample.json.gz")
    if err != nil {
        fmt.Println(err.Error())
        t.Fail()
    }
    r, err := gzip.NewReader(file)
    if err != nil {
        fmt.Println(err.Error())
        t.Fail()
    }
    sc := bufio.NewScanner(r)
    json.Unmarshal(sc.Bytes(), trieUnmarshal)

    // attempt via json Decoder
    b, err = ioutil.ReadFile("../resources/trieSample.json.gz")
    if err != nil {
        fmt.Println(err.Error())
        t.Fail()
    }
    bReader := bytes.NewReader(b)
    json.NewDecoder(bReader).Decode(trieDecoder)

    // spew.Dump shows that object is not populated
    spew.Dump(trieUnmarshal)
    spew.Dump(trieDecoder)
}

spew.Dump shows that the trieSlice Children array has all nil elements

Close the compressor before using the data. Decompress the data before using it. Don't chop it up with inappropriate use of bufio.Scanner.

var network bytes.Buffer
b, err := json.Marshal(trie)
if err != nil {
    fmt.Println("error in marshal ... ", err.Error())
    t.Fail()
}
w := gzip.NewWriter(&network)
w.Write(b)
w.Close()
err = ioutil.WriteFile("trieSample.json.gz", network.Bytes(), 0644)
if err != nil {
    log.Fatal(err)
}

trieDecoder := NewTrieSlice(charSet)

// attempt via json Unmarshal
file, err := os.Open("trieSample.json.gz")
if err != nil {
    log.Fatal(err)
}
r, err := gzip.NewReader(file)
if err != nil {
    log.Fatal(err)
}
err = json.NewDecoder(r).Decode(trieDecoder)
if err != nil {
    log.Fatal(err)
}
spew.Dump(trieDecoder)

https://play.golang.org/p/pYup3v8-f4c