improve file loading

Author: JojiiOfficial

dmfs/data.go

@@ -16,19 +16,29 @@ var data dataStruct
 
 // data provides the data for the fs
 type dataStruct struct {
+	// Lib and config stuff
 	mounter *Mounter
 	config  *dmConfig.Config
 	libdm   *libdatamanager.LibDM
 
+	// Request stuff + cache
+	// -- User attributes
 	userAttributes   *libdm.UserAttributeDataResponse
 	lastUserAttrLoad int64
+	// -- Files
+	filesCache   map[string][]libdm.FileResponseItem
+	lastFileload map[string]int64
 
+	// Global values
 	gid, uid uint32
 }
 
 func initData() {
 	data.gid = uint32(os.Getegid())
 	data.uid = uint32(os.Getuid())
+
+	data.filesCache = make(map[string][]libdm.FileResponseItem)
+	data.lastFileload = make(map[string]int64)
 }
 
 // load user attributes (namespaces, groups)
@@ -39,7 +49,7 @@ func (data *dataStruct) loadUserAttributes() error {
 	if time.Now().Unix()-5 > data.lastUserAttrLoad {
 		fmt.Println("reload")
 		var err error
-		data.userAttributes, err = data.libdm.GetUserAttributeData(1)
+		data.userAttributes, err = data.libdm.GetUserAttributeData()
 		if err != nil {
 			printResponseError(err, "loading user attributes")
 			return err
@@ -51,6 +61,37 @@ func (data *dataStruct) loadUserAttributes() error {
 	return nil
 }
 
+func (data *dataStruct) loadFiles(attributes libdatamanager.FileAttributes) ([]libdatamanager.FileResponseItem, error) {
+	lastLoad := data.getLastFileLoad(attributes.Namespace)
+
+	if time.Now().Unix()-5 > lastLoad {
+		fmt.Println("fresh file load")
+		resp, err := data.libdm.ListFiles("", 0, false, attributes, 0)
+		if err != nil {
+			return nil, err
+		}
+
+		// Set cache
+		data.filesCache[attributes.Namespace] = resp.Files
+		return resp.Files, nil
+	}
+
+	return data.filesCache[attributes.Namespace], nil
+}
+
+func (data *dataStruct) getLastFileLoad(namespace string) int64 {
+	v, h := data.lastFileload[namespace]
+	defer func() {
+		data.lastFileload[namespace] = time.Now().Unix()
+	}()
+
+	if !h {
+		return 0
+	}
+
+	return v
+}
+
 func (data *dataStruct) setUserAttr(inAttr *fuse.AttrOut) {
 	inAttr.Gid = data.gid
 	inAttr.Uid = data.uid

dmfs/fsFile.go

@@ -2,6 +2,7 @@ package dmfs
 
 import (
 	"context"
+	"fmt"
 	"syscall"
 
 	libdm "github.com/DataManager-Go/libdatamanager"
@@ -19,6 +20,7 @@ var _ = (fs.NodeGetattrer)((*fileInode)(nil))
 
 // Set file attributes
 func (fnode *fileInode) Getattr(ctx context.Context, f fs.FileHandle, out *fuse.AttrOut) syscall.Errno {
+	fmt.Println("getatr")
 	out.Size = uint64(fnode.file.Size)
 	out.Ctime = uint64(fnode.file.CreationDate.Unix())
 	out.Mtime = out.Ctime

dmfs/fsGroup.go

@@ -2,10 +2,13 @@ package dmfs
 
 import (
 	"context"
+	"fmt"
 	"syscall"
 
 	"github.com/DataManager-Go/libdatamanager"
+	libdm "github.com/DataManager-Go/libdatamanager"
 	"github.com/hanwen/go-fuse/v2/fs"
+	"github.com/hanwen/go-fuse/v2/fuse"
 )
 
 const (
@@ -15,44 +18,91 @@ const (
 )
 
 var (
-	_ = (fs.NodeOnAdder)((*groupInode)(nil))
+	_ = (fs.NodeReaddirer)((*groupNode)(nil))
+	_ = (fs.NodeLookuper)((*groupNode)(nil))
 )
 
-type groupInode struct {
+type groupNode struct {
 	fs.Inode
 
 	namespace            string
 	group                string
 	isNoGroupPlaceholder bool
+
+	files []libdm.FileResponseItem
 }
 
-// List files
-func (groupInode *groupInode) OnAdd(ctx context.Context) {
-	groupAdd := []string{groupInode.group}
+// Create a new group node
+func newGroupNode(namespace, group string) *groupNode {
+	return &groupNode{
+		namespace:            namespace,
+		group:                group,
+		isNoGroupPlaceholder: group == NoGroupFolder,
+	}
+}
 
-	// Check if group is no_group
-	if len(groupAdd) == 1 && groupAdd[0] == NoGroupFolder {
-		groupInode.isNoGroupPlaceholder = true
-		groupAdd = []string{}
+func (groupNode *groupNode) getRequestAttributes() libdatamanager.FileAttributes {
+	// Don't send any group to get
+	// all files without groups
+	reqGroup := []string{groupNode.group}
+	if groupNode.isNoGroupPlaceholder {
+		reqGroup = []string{}
 	}
 
-	files, err := data.libdm.ListFiles("", 0, false, libdatamanager.FileAttributes{
-		Groups:    groupAdd,
-		Namespace: groupInode.namespace,
-	}, 0)
+	return libdatamanager.FileAttributes{
+		Namespace: groupNode.namespace,
+		Groups:    reqGroup,
+	}
+}
 
+// List files in group
+func (groupNode *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) {
+	r := make([]fuse.DirEntry, 0)
+	files, err := data.loadFiles(groupNode.getRequestAttributes())
 	if err != nil {
-		printResponseError(err, "getting files for "+groupInode.group)
-		return
+		return nil, syscall.EIO
+	}
+
+	for i := range files {
+		r = append(r, fuse.DirEntry{
+			Mode: syscall.S_IFREG,
+			Name: files[i].Name,
+		})
+
+		fmt.Println(files[i].Name)
 	}
 
-	for _, file := range files.Files {
-		fileNode := groupInode.NewInode(ctx, &fileInode{
-			file: &file,
+	groupNode.files = files
+
+	return fs.NewListDirStream(r), 0
+}
+
+func (groupNode *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) {
+	file := groupNode.findFile(name)
+	if file == nil {
+		return nil, syscall.ENOENT
+	}
+
+	file.Attributes.Namespace = groupNode.namespace
+
+	child := groupNode.GetChild(name)
+	if child == nil {
+		child = groupNode.NewInode(ctx, &fileInode{
+			file: file,
 		}, fs.StableAttr{
 			Mode: syscall.S_IFREG,
 		})
+	}
 
-		groupInode.AddChild(file.Name, fileNode, true)
+	return child, 0
+}
+
+func (groupNode *groupNode) findFile(name string) *libdatamanager.FileResponseItem {
+	for i := range groupNode.files {
+		if groupNode.files[i].Name == name {
+			return &groupNode.files[i]
+		}
 	}
+
+	return nil
 }

dmfs/fsNamespace.go

@@ -94,11 +94,7 @@ func (nsNode *namespaceNode) Lookup(ctx context.Context, name string, out *fuse.
 
 	// create child if not found
 	if child == nil {
-		v := &groupInode{
-			group:                name,
-			isNoGroupPlaceholder: name == NoGroupFolder,
-			namespace:            nsNode.nsInfo.Name,
-		}
+		v := newGroupNode(nsNode.nsInfo.Name, name)
 
 		child = nsNode.NewInode(ctx, v, fs.StableAttr{
 			Mode: syscall.S_IFDIR,
@@ -155,7 +151,7 @@ func (nsNode *namespaceNode) Mkdir(ctx context.Context, name string, mode uint32
 	}
 
 	nsNode.nsInfo.Groups = append(nsNode.nsInfo.Groups, name)
-	node := nsNode.NewInode(ctx, &groupInode{
+	node := nsNode.NewInode(ctx, &groupNode{
 		group:     name,
 		namespace: nsNode.nsInfo.Name,
 	}, fs.StableAttr{

go.sum

@@ -3,6 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/DataManager-Go/libdatamanager v1.0.10/go.mod h1:iLQrv9tvUkcx9nA+x51eUR9juV0JgIOnFrhPISxWWUE=
 github.com/DataManager-Go/libdatamanager v1.2.3 h1:a6qTHmrFG6tpBecMM4XxEtV5sqerjnRPS52NMPdHyKM=
 github.com/DataManager-Go/libdatamanager v1.2.3/go.mod h1:s2ywdIZpWEfgxJ7iMtCOdLCNaHVnO8dr7e9cE2NzgL4=
+github.com/DataManager-Go/libdatamanager v1.2.4 h1:K5AHX32XPLO4RyU3WrE0tBSYVDAaWWrwuuKBMq5RMVs=
+github.com/DataManager-Go/libdatamanager v1.2.4/go.mod h1:s2ywdIZpWEfgxJ7iMtCOdLCNaHVnO8dr7e9cE2NzgL4=
 github.com/DataManager-Go/libdatamanager/config v0.0.0-20200418200810-31f4c1161174 h1:exehOGqNqQ8/g5R4PPP/Qcok6kt4Q+os6b0+rEcsWlU=
 github.com/DataManager-Go/libdatamanager/config v0.0.0-20200418200810-31f4c1161174/go.mod h1:RfoOPe8dQgubuBUmtqEYu1Ny0fT+htZH/ArrEl0dcHQ=
 github.com/JojiiOfficial/configService v0.0.0-20200219132202-6e71512e2e28 h1:nYoIExG+Z/gSLS9Jbpu6lnrh+m6e9gTxQfGhanTsExE=