Index: VERSION
==================================================================
--- VERSION
+++ VERSION
@@ -1,1 +1,1 @@
-0.0.11
+0.0.12
ADDED cmd/fd_limit.go
Index: cmd/fd_limit.go
==================================================================
--- /dev/null
+++ cmd/fd_limit.go
@@ -0,0 +1,15 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2021 Detlef Stern
+//
+// This file is part of zettelstore.
+//
+// Zettelstore is licensed under the latest version of the EUPL (European Union
+// Public License). Please see file LICENSE.txt for your rights and obligations
+// under this license.
+//-----------------------------------------------------------------------------
+
+// +build !darwin
+
+package cmd
+
+func raiseFdLimit() error { return nil }
ADDED cmd/fd_limit_raise.go
Index: cmd/fd_limit_raise.go
==================================================================
--- /dev/null
+++ cmd/fd_limit_raise.go
@@ -0,0 +1,47 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2021 Detlef Stern
+//
+// This file is part of zettelstore.
+//
+// Zettelstore is licensed under the latest version of the EUPL (European Union
+// Public License). Please see file LICENSE.txt for your rights and obligations
+// under this license.
+//-----------------------------------------------------------------------------
+
+// +build darwin
+
+package cmd
+
+import (
+ "log"
+ "syscall"
+)
+
+const minFiles = 1048576
+
+func raiseFdLimit() error {
+ var rLimit syscall.Rlimit
+ err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+ if err != nil {
+ return err
+ }
+ if rLimit.Cur >= minFiles {
+ return nil
+ }
+ rLimit.Cur = minFiles
+ if rLimit.Cur > rLimit.Max {
+ rLimit.Cur = rLimit.Max
+ }
+ err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+ if err != nil {
+ return err
+ }
+ err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+ if err != nil {
+ return err
+ }
+ if rLimit.Cur < minFiles {
+ log.Printf("Make sure you have no more than %d files in all your places if you enabled notification\n", rLimit.Cur)
+ }
+ return nil
+}
Index: cmd/main.go
==================================================================
--- cmd/main.go
+++ cmd/main.go
@@ -12,10 +12,11 @@
import (
"context"
"flag"
"fmt"
+ "log"
"os"
"strings"
"zettelstore.de/z/config/runtime"
"zettelstore.de/z/config/startup"
@@ -125,24 +126,28 @@
func setupOperations(cfg *meta.Meta, withPlaces, simple bool) error {
var mgr place.Manager
var idx index.Indexer
if withPlaces {
+ err := raiseFdLimit()
+ if err != nil {
+ log.Println("Raising some limitions did not work:", err)
+ log.Println("Prepare to encounter errors. Most of them can be mitigated. See the manual for details")
+ cfg.Set(startup.KeyDefaultDirPlaceType, startup.ValueDirPlaceTypeSimple)
+ }
+ startup.SetupStartupConfig(cfg)
idx = indexer.New()
filter := index.NewMetaFilter(idx)
- p, err := manager.New(getPlaces(cfg), cfg.GetBool(startup.KeyReadOnlyMode), filter)
+ mgr, err = manager.New(getPlaces(cfg), cfg.GetBool(startup.KeyReadOnlyMode), filter)
if err != nil {
return err
}
- mgr = p
+ } else {
+ startup.SetupStartupConfig(cfg)
}
- err := startup.SetupStartup(cfg, mgr, idx, simple)
- if err != nil {
- fmt.Fprintln(os.Stderr, "Unable to connect to specified places")
- return err
- }
+ startup.SetupStartupService(mgr, idx, simple)
if withPlaces {
if err := mgr.Start(context.Background()); err != nil {
fmt.Fprintln(os.Stderr, "Unable to start zettel place")
return err
}
Index: config/startup/startup.go
==================================================================
--- config/startup/startup.go
+++ config/startup/startup.go
@@ -22,46 +22,56 @@
"zettelstore.de/z/index"
"zettelstore.de/z/place"
)
var config struct {
- simple bool // was started without run command
- verbose bool
- readonlyMode bool
- urlPrefix string
- listenAddress string
- owner id.Zid
- withAuth bool
- secret []byte
- insecCookie bool
- persistCookie bool
- htmlLifetime time.Duration
- apiLifetime time.Duration
- manager place.Manager
- indexer index.Indexer
+ // Set in SetupStartupConfig
+ verbose bool
+ readonlyMode bool
+ urlPrefix string
+ listenAddress string
+ defaultDirPlaceType string
+ owner id.Zid
+ withAuth bool
+ secret []byte
+ insecCookie bool
+ persistCookie bool
+ htmlLifetime time.Duration
+ apiLifetime time.Duration
+
+ // Set in SetupStartupService
+ simple bool // was started without run command
+ manager place.Manager
+ indexer index.Indexer
}
// Predefined keys for startup zettel
const (
- KeyInsecureCookie = "insecure-cookie"
- KeyListenAddress = "listen-addr"
- KeyOwner = "owner"
- KeyPersistentCookie = "persistent-cookie"
- KeyPlaceOneURI = "place-1-uri"
- KeyReadOnlyMode = "read-only-mode"
- KeyTokenLifetimeHTML = "token-lifetime-html"
- KeyTokenLifetimeAPI = "token-lifetime-api"
- KeyURLPrefix = "url-prefix"
- KeyVerbose = "verbose"
+ KeyDefaultDirPlaceType = "default-dir-place-type"
+ KeyInsecureCookie = "insecure-cookie"
+ KeyListenAddress = "listen-addr"
+ KeyOwner = "owner"
+ KeyPersistentCookie = "persistent-cookie"
+ KeyPlaceOneURI = "place-1-uri"
+ KeyReadOnlyMode = "read-only-mode"
+ KeyTokenLifetimeHTML = "token-lifetime-html"
+ KeyTokenLifetimeAPI = "token-lifetime-api"
+ KeyURLPrefix = "url-prefix"
+ KeyVerbose = "verbose"
+)
+
+// Important values for some keys.
+const (
+ ValueDirPlaceTypeNotify = "notify"
+ ValueDirPlaceTypeSimple = "simple"
)
-// SetupStartup initializes the startup data.
-func SetupStartup(cfg *meta.Meta, manager place.Manager, idx index.Indexer, simple bool) error {
+// SetupStartupConfig initializes the startup data with content of config file.
+func SetupStartupConfig(cfg *meta.Meta) {
if config.urlPrefix != "" {
panic("startup.config already set")
}
- config.simple = simple
config.verbose = cfg.GetBool(KeyVerbose)
config.readonlyMode = cfg.GetBool(KeyReadOnlyMode)
config.urlPrefix = cfg.GetDefault(KeyURLPrefix, "/")
if prefix, ok := cfg.Get(KeyURLPrefix); ok &&
len(prefix) > 0 && prefix[0] == '/' && prefix[len(prefix)-1] == '/' {
@@ -71,10 +81,21 @@
}
if val, ok := cfg.Get(KeyListenAddress); ok {
config.listenAddress = val // TODO: check for valid string
} else {
config.listenAddress = "127.0.0.1:23123"
+ }
+ if defaultType, ok := cfg.Get(KeyDefaultDirPlaceType); ok {
+ switch defaultType {
+ case ValueDirPlaceTypeNotify:
+ case ValueDirPlaceTypeSimple:
+ default:
+ defaultType = ValueDirPlaceTypeNotify
+ }
+ config.defaultDirPlaceType = defaultType
+ } else {
+ config.defaultDirPlaceType = ValueDirPlaceTypeNotify
}
config.owner = id.Invalid
if owner, ok := cfg.Get(KeyOwner); ok {
if zid, err := id.Parse(owner); err == nil {
config.owner = zid
@@ -88,14 +109,20 @@
config.htmlLifetime = getDuration(
cfg, KeyTokenLifetimeHTML, 1*time.Hour, 1*time.Minute, 30*24*time.Hour)
config.apiLifetime = getDuration(
cfg, KeyTokenLifetimeAPI, 10*time.Minute, 0, 1*time.Hour)
}
+}
+
+// SetupStartupService initializes the startup data with internal services.
+func SetupStartupService(manager place.Manager, idx index.Indexer, simple bool) {
+ if config.urlPrefix == "" {
+ panic("startup.config not set")
+ }
config.simple = simple && !config.withAuth
config.manager = manager
config.indexer = idx
- return nil
}
func calcSecret(cfg *meta.Meta) []byte {
h := fnv.New128()
if secret, ok := cfg.Get("secret"); ok {
@@ -141,10 +168,13 @@
func URLPrefix() string { return config.urlPrefix }
// ListenAddress returns the string that specifies the the network card and the ip port
// where the server listens for requests
func ListenAddress() string { return config.listenAddress }
+
+// DefaultDirPlaceType returns the default value for a directory place type.
+func DefaultDirPlaceType() string { return config.defaultDirPlaceType }
// WithAuth returns true if user authentication is enabled.
func WithAuth() bool { return config.withAuth }
// SecureCookie returns whether the web app should set cookies to secure mode.
Index: docs/manual/00001004010000.zettel
==================================================================
--- docs/manual/00001004010000.zettel
+++ docs/manual/00001004010000.zettel
@@ -13,10 +13,15 @@
The file for start-up configuration must be created via a text editor in advance.
The syntax of the configuration file is the same as for any zettel metadata.
The following keys are supported:
+; [!default-dir-place-type]''default-dir-place-type''
+: Specifies the default value for the (sub-) type of [[directory places|00001004011400#type]].
+ Zettel are typically stored in such places.
+
+ Default: ''notify''
; [!insecure-cookie]''insecure-cookie''
: Must be set to ''true'', if authentication is enabled and Zettelstore is not accessible not via HTTPS (but via HTTP).
Otherwise web browser are free to ignore the authentication cookie.
Default: ''false''
Index: docs/manual/00001004011400.zettel
==================================================================
--- docs/manual/00001004011400.zettel
+++ docs/manual/00001004011400.zettel
@@ -1,27 +1,45 @@
id: 00001004011400
title: Configure file directory places
+role: manual
tags: #configuration #manual #zettelstore
syntax: zmk
-role: manual
Under certain circumstances, it is preferable to further configure a file directory place.
This is done by appending query parameters after the base place URI ''dir:\//DIR''.
The following parameters are supported:
|= Parameter:|Description|Default value:|
+|type|(Sub-) Type of the directory service|(value of ''[[default-dir-place-type|00001004010000#default-dir-place-type]]'')
|rescan|Time (in seconds) after which the directory should be scanned fully|600
-|worker|Number of worker that can access the directory in parallel|17
+|worker|Number of worker that can access the directory in parallel|(depends on type)
|readonly|Allow only operations that do not change a zettel or create a new zettel|n/a
+=== Type
+On some operating systems, Zettelstore tries to detect changes to zettel files outside of Zettelstore's control[^This includes Linux, Windows, and macOS.].
+On other operating systems, this may be not possible, due to technical limitations.
+Automatic detection of external changes is also not possible, if zettel files are placed on an external service, such as a file server accessed via SMD/CIFS or NFS.
+
+To cope with this uncertainty, Zettelstore provides various internal implementations of a directory place.
+The default values should match the needs of different users, as explained in the [[installation part|00001003000000]] of this manual.
+The following values are supported:
+
+; simple
+: Is not able to detect external changes.
+ Works on all platforms.
+ Is a little slower than other implementations (up to three times slower).
+; notify
+: Automatically detect external changes.
+ Tries to optimize performance, at a little cost of main memory (RAM).
+
=== Rescan
-On most platforms, Zettelstore automatically detects changes to zettel files that originates from other software[^This includes most Linux distributions, macOS, and Windows].
+When the parameter ''type'' is set to ""notify"", Zettelstore automatically detects changes to zettel files that originates from other software.
It is done on a ""best-effort"" basis.
Under certain circumstances it is possible that Zettelstore does not detect a change done by another software.
-To cope with this unlikely, but still possible event, Zettelstore re-scans periodically the file diectory.
+To cope with this unlikely, but still possible event, Zettelstore re-scans periodically the file directory.
The time interval is configured by the ''rescan'' parameter, e.g.
```
place-1-uri: dir:///home/zettel?rescan=300
```
This makes Zettelstore to re-scan the directory ''/home/zettel/'' every 300 seconds, i.e. 5 minutes.
@@ -31,28 +49,33 @@
Please note that a directory re-scan invalidates all internal data of a Zettelstore.
It might trigger a re-build of the backlink database (and other internal databases).
Therefore a large value if preferred.
+This value is ignored for other directory place type, such as ""simple"".
+
=== Worker
Internally, Zettelstore parallels concurrent requests for a zettel or its metadata.
The number of parallel activities is configured by the ''worker'' parameter.
-Its default value 17 is a good compromise when you think about the high variability of possible Zettelstore environments.
A computer contains a limited number of internal processing units (CPU).
Its number ranges from 1 to (currently) 128, e.g. in bigger server environments.
Zettelstore typically runs on a system with 1 to 8 CPUs.
Access to zettel file is ultimately managed by the underlying operating system.
-Depending on the hardware, only a limited number of parallel accesses are desireable.
-Since Zettelstore is a single-user software, the value 17 is quite reasonable, even for higher use.
+Depending on the hardware and on the type of the directory place, only a limited number of parallel accesses are desirable.
On smaller hardware[^In comparison to a normal desktop or laptop computer], such as the [[Raspberry Zero|https://www.raspberrypi.org/products/raspberry-pi-zero/]], a smaller value might be appropriate.
Every worker needs some amount of main memory (RAM) and some amount of processing power.
On bigger hardware, with some fast file services, a bigger value could result in higher performance, if needed.
+For a directory place of type ""notify"", the default value is: 7.
+The directory place type ""simple"" limits the value to a maximum of 1, i.e. no concurrency is possible with this type of directory place.
+
+For various reasons, the value should be a prime number, with a maximum value of 1499.
+
=== Readonly
Sometimes you may want to provide zettel from a file directory place, but you want to disallow any changes.
If you provide the query parameter ''readonly'' (with or without a corresponding value), the place will disallow any changes.
```
place-1-uri: dir:///home/zettel?readonly
```
If you put the whole Zettelstore in [[read-only|00001004010000]] [[mode|00001004051000]], all configured file directory places will be in read-only mode too, even if not explicitly configured.
Index: go.mod
==================================================================
--- go.mod
+++ go.mod
@@ -3,10 +3,10 @@
go 1.16
require (
github.com/fsnotify/fsnotify v1.4.9
github.com/pascaldekloe/jwt v1.10.0
- github.com/yuin/goldmark v1.3.3
+ github.com/yuin/goldmark v1.3.5
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221
golang.org/x/text v0.3.6
)
Index: go.sum
==================================================================
--- go.sum
+++ go.sum
@@ -1,11 +1,11 @@
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/pascaldekloe/jwt v1.10.0 h1:ktcIUV4TPvh404R5dIBEnPCsSwj0sqi3/0+XafE5gJs=
github.com/pascaldekloe/jwt v1.10.0/go.mod h1:TKhllgThT7TOP5rGr2zMLKEDZRAgJfBbtKyVeRsNB9A=
-github.com/yuin/goldmark v1.3.3 h1:37BdQwPx8VOSic8eDSWee6QL9mRpZRm9VJp/QugNrW0=
-github.com/yuin/goldmark v1.3.3/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Index: place/dirplace/directory/directory.go
==================================================================
--- place/dirplace/directory/directory.go
+++ place/dirplace/directory/directory.go
@@ -6,115 +6,48 @@
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------
-// Package directory manages the directory part of a dirstore.
-package directory
-
-import (
- "time"
-
- "zettelstore.de/z/domain/id"
- "zettelstore.de/z/place/change"
-)
-
-// Service specifies a directory scan service.
-type Service struct {
- dirPath string
- rescanTime time.Duration
- done chan struct{}
- cmds chan dirCmd
- infos chan<- change.Info
-}
-
-// NewService creates a new directory service.
-func NewService(directoryPath string, rescanTime time.Duration, chci chan<- change.Info) *Service {
- srv := &Service{
- dirPath: directoryPath,
- rescanTime: rescanTime,
- cmds: make(chan dirCmd),
- infos: chci,
- }
- return srv
-}
-
-// Start makes the directory service operational.
-func (srv *Service) Start() {
- tick := make(chan struct{})
- rawEvents := make(chan *fileEvent)
- events := make(chan *fileEvent)
-
- ready := make(chan int)
- go srv.directoryService(events, ready)
- go collectEvents(events, rawEvents)
- go watchDirectory(srv.dirPath, rawEvents, tick)
-
- if srv.done != nil {
- panic("src.done already set")
- }
- srv.done = make(chan struct{})
- go ping(tick, srv.rescanTime, srv.done)
- <-ready
-}
-
-// Stop stops the directory service.
-func (srv *Service) Stop() {
- close(srv.done)
- srv.done = nil
-}
-
-func (srv *Service) notifyChange(reason change.Reason, zid id.Zid) {
- if chci := srv.infos; chci != nil {
- chci <- change.Info{Reason: reason, Zid: zid}
- }
-}
-
-// NumEntries returns the number of managed zettel.
-func (srv *Service) NumEntries() int {
- resChan := make(chan resNumEntries)
- srv.cmds <- &cmdNumEntries{resChan}
- return <-resChan
-}
-
-// GetEntries returns an unsorted list of all current directory entries.
-func (srv *Service) GetEntries() []Entry {
- resChan := make(chan resGetEntries)
- srv.cmds <- &cmdGetEntries{resChan}
- return <-resChan
-}
-
-// GetEntry returns the entry with the specified zettel id. If there is no such
-// zettel id, an empty entry is returned.
-func (srv *Service) GetEntry(zid id.Zid) Entry {
- resChan := make(chan resGetEntry)
- srv.cmds <- &cmdGetEntry{zid, resChan}
- return <-resChan
-}
-
-// GetNew returns an entry with a new zettel id.
-func (srv *Service) GetNew() Entry {
- resChan := make(chan resNewEntry)
- srv.cmds <- &cmdNewEntry{resChan}
- return <-resChan
-}
-
-// UpdateEntry notifies the directory of an updated entry.
-func (srv *Service) UpdateEntry(entry *Entry) {
- resChan := make(chan struct{})
- srv.cmds <- &cmdUpdateEntry{entry, resChan}
- <-resChan
-}
-
-// RenameEntry notifies the directory of an renamed entry.
-func (srv *Service) RenameEntry(curEntry, newEntry *Entry) error {
- resChan := make(chan resRenameEntry)
- srv.cmds <- &cmdRenameEntry{curEntry, newEntry, resChan}
- return <-resChan
-}
-
-// DeleteEntry removes a zettel id from the directory of entries.
-func (srv *Service) DeleteEntry(zid id.Zid) {
- resChan := make(chan struct{})
- srv.cmds <- &cmdDeleteEntry{zid, resChan}
- <-resChan
+// Package directory manages the directory interface of a dirstore.
+package directory
+
+import "zettelstore.de/z/domain/id"
+
+// Service is the interface of a directory service.
+type Service interface {
+ Start() error
+ Stop() error
+ NumEntries() (int, error)
+ GetEntries() ([]*Entry, error)
+ GetEntry(zid id.Zid) (*Entry, error)
+ GetNew() (*Entry, error)
+ UpdateEntry(entry *Entry) error
+ RenameEntry(curEntry, newEntry *Entry) error
+ DeleteEntry(zid id.Zid) error
+}
+
+// MetaSpec defines all possibilities where meta data can be stored.
+type MetaSpec int
+
+// Constants for MetaSpec
+const (
+ _ MetaSpec = iota
+ MetaSpecNone // no meta information
+ MetaSpecFile // meta information is in meta file
+ MetaSpecHeader // meta information is in header
+)
+
+// Entry stores everything for a directory entry.
+type Entry struct {
+ Zid id.Zid
+ MetaSpec MetaSpec // location of meta information
+ MetaPath string // file path of meta information
+ ContentPath string // file path of zettel content
+ ContentExt string // (normalized) file extension of zettel content
+ Duplicates bool // multiple content files
+}
+
+// IsValid checks whether the entry is valid.
+func (e *Entry) IsValid() bool {
+ return e != nil && e.Zid.IsValid()
}
DELETED place/dirplace/directory/entry.go
Index: place/dirplace/directory/entry.go
==================================================================
--- place/dirplace/directory/entry.go
+++ /dev/null
@@ -1,40 +0,0 @@
-//-----------------------------------------------------------------------------
-// Copyright (c) 2020-2021 Detlef Stern
-//
-// This file is part of zettelstore.
-//
-// Zettelstore is licensed under the latest version of the EUPL (European Union
-// Public License). Please see file LICENSE.txt for your rights and obligations
-// under this license.
-//-----------------------------------------------------------------------------
-
-// Package directory manages the directory part of a dirstore.
-package directory
-
-import "zettelstore.de/z/domain/id"
-
-// MetaSpec defines all possibilities where meta data can be stored.
-type MetaSpec int
-
-// Constants for MetaSpec
-const (
- MetaSpecUnknown MetaSpec = iota
- MetaSpecNone // no meta information
- MetaSpecFile // meta information is in meta file
- MetaSpecHeader // meta information is in header
-)
-
-// Entry stores everything for a directory entry.
-type Entry struct {
- Zid id.Zid
- MetaSpec MetaSpec // location of meta information
- MetaPath string // file path of meta information
- ContentPath string // file path of zettel content
- ContentExt string // (normalized) file extension of zettel content
- Duplicates bool // multiple content files
-}
-
-// IsValid checks whether the entry is valid.
-func (e *Entry) IsValid() bool {
- return e.Zid.IsValid()
-}
DELETED place/dirplace/directory/service.go
Index: place/dirplace/directory/service.go
==================================================================
--- place/dirplace/directory/service.go
+++ /dev/null
@@ -1,255 +0,0 @@
-//-----------------------------------------------------------------------------
-// Copyright (c) 2020-2021 Detlef Stern
-//
-// This file is part of zettelstore.
-//
-// Zettelstore is licensed under the latest version of the EUPL (European Union
-// Public License). Please see file LICENSE.txt for your rights and obligations
-// under this license.
-//-----------------------------------------------------------------------------
-
-// Package directory manages the directory part of a directory place.
-package directory
-
-import (
- "log"
- "time"
-
- "zettelstore.de/z/domain/id"
- "zettelstore.de/z/place"
- "zettelstore.de/z/place/change"
-)
-
-// ping sends every tick a signal to reload the directory list
-func ping(tick chan<- struct{}, rescanTime time.Duration, done <-chan struct{}) {
- ticker := time.NewTicker(rescanTime)
- defer close(tick)
- for {
- select {
- case _, ok := <-ticker.C:
- if !ok {
- return
- }
- tick <- struct{}{}
- case _, ok := <-done:
- if !ok {
- ticker.Stop()
- return
- }
- }
- }
-}
-
-func newEntry(ev *fileEvent) *Entry {
- de := new(Entry)
- de.Zid = ev.zid
- updateEntry(de, ev)
- return de
-}
-
-func updateEntry(de *Entry, ev *fileEvent) {
- if ev.ext == "meta" {
- de.MetaSpec = MetaSpecFile
- de.MetaPath = ev.path
- return
- }
- if de.ContentExt != "" && de.ContentExt != ev.ext {
- de.Duplicates = true
- return
- }
- if de.MetaSpec != MetaSpecFile {
- if ev.ext == "zettel" {
- de.MetaSpec = MetaSpecHeader
- } else {
- de.MetaSpec = MetaSpecNone
- }
- }
- de.ContentPath = ev.path
- de.ContentExt = ev.ext
-}
-
-type dirMap map[id.Zid]*Entry
-
-func dirMapUpdate(dm dirMap, ev *fileEvent) {
- de := dm[ev.zid]
- if de == nil {
- dm[ev.zid] = newEntry(ev)
- return
- }
- updateEntry(de, ev)
-}
-
-func deleteFromMap(dm dirMap, ev *fileEvent) {
- if ev.ext == "meta" {
- if entry, ok := dm[ev.zid]; ok {
- if entry.MetaSpec == MetaSpecFile {
- entry.MetaSpec = MetaSpecNone
- return
- }
- }
- }
- delete(dm, ev.zid)
-}
-
-// directoryService is the main service.
-func (srv *Service) directoryService(events <-chan *fileEvent, ready chan<- int) {
- curMap := make(dirMap)
- var newMap dirMap
- for {
- select {
- case ev, ok := <-events:
- if !ok {
- return
- }
- switch ev.status {
- case fileStatusReloadStart:
- newMap = make(dirMap)
- case fileStatusReloadEnd:
- curMap = newMap
- newMap = nil
- if ready != nil {
- ready <- len(curMap)
- close(ready)
- ready = nil
- }
- srv.notifyChange(change.OnReload, id.Invalid)
- case fileStatusError:
- log.Println("DIRPLACE", "ERROR", ev.err)
- case fileStatusUpdate:
- srv.processFileUpdateEvent(ev, curMap, newMap)
- case fileStatusDelete:
- srv.processFileDeleteEvent(ev, curMap, newMap)
- }
- case cmd, ok := <-srv.cmds:
- if ok {
- cmd.run(curMap)
- }
- }
- }
-}
-
-func (srv *Service) processFileUpdateEvent(ev *fileEvent, curMap, newMap dirMap) {
- if newMap != nil {
- dirMapUpdate(newMap, ev)
- } else {
- dirMapUpdate(curMap, ev)
- srv.notifyChange(change.OnUpdate, ev.zid)
- }
-}
-
-func (srv *Service) processFileDeleteEvent(ev *fileEvent, curMap, newMap dirMap) {
- if newMap != nil {
- deleteFromMap(newMap, ev)
- } else {
- deleteFromMap(curMap, ev)
- srv.notifyChange(change.OnDelete, ev.zid)
- }
-}
-
-type dirCmd interface {
- run(m dirMap)
-}
-
-type cmdNumEntries struct {
- result chan<- resNumEntries
-}
-type resNumEntries = int
-
-func (cmd *cmdNumEntries) run(m dirMap) {
- cmd.result <- len(m)
-}
-
-type cmdGetEntries struct {
- result chan<- resGetEntries
-}
-type resGetEntries []Entry
-
-func (cmd *cmdGetEntries) run(m dirMap) {
- res := make([]Entry, 0, len(m))
- for _, de := range m {
- res = append(res, *de)
- }
- cmd.result <- res
-}
-
-type cmdGetEntry struct {
- zid id.Zid
- result chan<- resGetEntry
-}
-type resGetEntry = Entry
-
-func (cmd *cmdGetEntry) run(m dirMap) {
- entry := m[cmd.zid]
- if entry == nil {
- cmd.result <- Entry{Zid: id.Invalid}
- } else {
- cmd.result <- *entry
- }
-}
-
-type cmdNewEntry struct {
- result chan<- resNewEntry
-}
-type resNewEntry = Entry
-
-func (cmd *cmdNewEntry) run(m dirMap) {
- zid := id.New(false)
- if _, ok := m[zid]; !ok {
- entry := &Entry{Zid: zid, MetaSpec: MetaSpecUnknown}
- m[zid] = entry
- cmd.result <- *entry
- return
- }
- for {
- zid = id.New(true)
- if _, ok := m[zid]; !ok {
- entry := &Entry{Zid: zid, MetaSpec: MetaSpecUnknown}
- m[zid] = entry
- cmd.result <- *entry
- return
- }
- // TODO: do not wait here, but in a non-blocking goroutine.
- time.Sleep(100 * time.Millisecond)
- }
-}
-
-type cmdUpdateEntry struct {
- entry *Entry
- result chan<- struct{}
-}
-
-func (cmd *cmdUpdateEntry) run(m dirMap) {
- entry := *cmd.entry
- m[entry.Zid] = &entry
- cmd.result <- struct{}{}
-}
-
-type cmdRenameEntry struct {
- curEntry *Entry
- newEntry *Entry
- result chan<- resRenameEntry
-}
-
-type resRenameEntry = error
-
-func (cmd *cmdRenameEntry) run(m dirMap) {
- newEntry := *cmd.newEntry
- newZid := newEntry.Zid
- if _, found := m[newZid]; found {
- cmd.result <- &place.ErrInvalidID{Zid: newZid}
- return
- }
- delete(m, cmd.curEntry.Zid)
- m[newZid] = &newEntry
- cmd.result <- nil
-}
-
-type cmdDeleteEntry struct {
- zid id.Zid
- result chan<- struct{}
-}
-
-func (cmd *cmdDeleteEntry) run(m dirMap) {
- delete(m, cmd.zid)
- cmd.result <- struct{}{}
-}
DELETED place/dirplace/directory/watch.go
Index: place/dirplace/directory/watch.go
==================================================================
--- place/dirplace/directory/watch.go
+++ /dev/null
@@ -1,300 +0,0 @@
-//-----------------------------------------------------------------------------
-// Copyright (c) 2020-2021 Detlef Stern
-//
-// This file is part of zettelstore.
-//
-// Zettelstore is licensed under the latest version of the EUPL (European Union
-// Public License). Please see file LICENSE.txt for your rights and obligations
-// under this license.
-//-----------------------------------------------------------------------------
-
-// Package directory manages the directory part of a directory place.
-package directory
-
-import (
- "os"
- "path/filepath"
- "regexp"
- "time"
-
- "github.com/fsnotify/fsnotify"
-
- "zettelstore.de/z/domain/id"
-)
-
-var validFileName = regexp.MustCompile(`^(\d{14}).*(\.(.+))$`)
-
-func matchValidFileName(name string) []string {
- return validFileName.FindStringSubmatch(name)
-}
-
-type fileStatus int
-
-const (
- fileStatusNone fileStatus = iota
- fileStatusReloadStart
- fileStatusReloadEnd
- fileStatusError
- fileStatusUpdate
- fileStatusDelete
-)
-
-type fileEvent struct {
- status fileStatus
- path string // Full file path
- zid id.Zid
- ext string // File extension
- err error // Error if Status == fileStatusError
-}
-
-type sendResult int
-
-const (
- sendDone sendResult = iota
- sendReload
- sendExit
-)
-
-func watchDirectory(directory string, events chan<- *fileEvent, tick <-chan struct{}) {
- defer close(events)
-
- var watcher *fsnotify.Watcher
- defer func() {
- if watcher != nil {
- watcher.Close()
- }
- }()
-
- sendEvent := func(ev *fileEvent) sendResult {
- select {
- case events <- ev:
- case _, ok := <-tick:
- if ok {
- return sendReload
- }
- return sendExit
- }
- return sendDone
- }
-
- sendError := func(err error) sendResult {
- return sendEvent(&fileEvent{status: fileStatusError, err: err})
- }
-
- sendFileEvent := func(status fileStatus, path string, match []string) sendResult {
- zid, err := id.Parse(match[1])
- if err != nil {
- return sendDone
- }
- event := &fileEvent{
- status: status,
- path: path,
- zid: zid,
- ext: match[3],
- }
- return sendEvent(event)
- }
-
- reloadStartEvent := &fileEvent{status: fileStatusReloadStart}
- reloadEndEvent := &fileEvent{status: fileStatusReloadEnd}
- reloadFiles := func() bool {
- entries, err := os.ReadDir(directory)
- if err != nil {
- if res := sendError(err); res != sendDone {
- return res == sendReload
- }
- return true
- }
-
- if res := sendEvent(reloadStartEvent); res != sendDone {
- return res == sendReload
- }
-
- if watcher != nil {
- watcher.Close()
- }
- watcher, err = fsnotify.NewWatcher()
- if err != nil {
- if res := sendError(err); res != sendDone {
- return res == sendReload
- }
- }
-
- for _, entry := range entries {
- if entry.IsDir() {
- continue
- }
- if info, err1 := entry.Info(); err1 != nil || !info.Mode().IsRegular() {
- continue
- }
- name := entry.Name()
- match := matchValidFileName(name)
- if len(match) > 0 {
- path := filepath.Join(directory, name)
- if res := sendFileEvent(fileStatusUpdate, path, match); res != sendDone {
- return res == sendReload
- }
- }
- }
-
- if watcher != nil {
- err = watcher.Add(directory)
- if err != nil {
- if res := sendError(err); res != sendDone {
- return res == sendReload
- }
- }
- }
- if res := sendEvent(reloadEndEvent); res != sendDone {
- return res == sendReload
- }
- return true
- }
-
- handleEvents := func() bool {
- const createOps = fsnotify.Create | fsnotify.Write
- const deleteOps = fsnotify.Remove | fsnotify.Rename
-
- for {
- select {
- case wevent, ok := <-watcher.Events:
- if !ok {
- return false
- }
- path := filepath.Clean(wevent.Name)
- match := matchValidFileName(filepath.Base(path))
- if len(match) == 0 {
- continue
- }
- if wevent.Op&createOps != 0 {
- if fi, err := os.Lstat(path); err != nil || !fi.Mode().IsRegular() {
- continue
- }
- if res := sendFileEvent(
- fileStatusUpdate, path, match); res != sendDone {
- return res == sendReload
- }
- }
- if wevent.Op&deleteOps != 0 {
- if res := sendFileEvent(
- fileStatusDelete, path, match); res != sendDone {
- return res == sendReload
- }
- }
- case err, ok := <-watcher.Errors:
- if !ok {
- return false
- }
- if res := sendError(err); res != sendDone {
- return res == sendReload
- }
- case _, ok := <-tick:
- return ok
- }
- }
- }
-
- for {
- if !reloadFiles() {
- return
- }
- if watcher == nil {
- if _, ok := <-tick; !ok {
- return
- }
- } else {
- if !handleEvents() {
- return
- }
- }
- }
-}
-
-func sendCollectedEvents(out chan<- *fileEvent, events []*fileEvent) {
- for _, ev := range events {
- if ev.status != fileStatusNone {
- out <- ev
- }
- }
-}
-
-func addEvent(events []*fileEvent, ev *fileEvent) []*fileEvent {
- switch ev.status {
- case fileStatusNone:
- return events
- case fileStatusReloadStart:
- events = events[0:0]
- case fileStatusUpdate, fileStatusDelete:
- if len(events) > 0 && mergeEvents(events, ev) {
- return events
- }
- }
- return append(events, ev)
-}
-
-func mergeEvents(events []*fileEvent, ev *fileEvent) bool {
- for i := len(events) - 1; i >= 0; i-- {
- oev := events[i]
- switch oev.status {
- case fileStatusReloadStart, fileStatusReloadEnd:
- return false
- case fileStatusUpdate, fileStatusDelete:
- if ev.path == oev.path {
- if ev.status == oev.status {
- return true
- }
- oev.status = fileStatusNone
- return false
- }
- }
- }
- return false
-}
-
-func collectEvents(out chan<- *fileEvent, in <-chan *fileEvent) {
- defer close(out)
-
- var sendTime time.Time
- sendTimeSet := false
- ticker := time.NewTicker(500 * time.Millisecond)
- defer ticker.Stop()
-
- events := make([]*fileEvent, 0, 32)
- buffer := false
- for {
- select {
- case ev, ok := <-in:
- if !ok {
- sendCollectedEvents(out, events)
- return
- }
- if ev.status == fileStatusReloadStart {
- buffer = false
- events = events[0:0]
- }
- if buffer {
- if !sendTimeSet {
- sendTime = time.Now().Add(1500 * time.Millisecond)
- sendTimeSet = true
- }
- events = addEvent(events, ev)
- if len(events) > 1024 {
- sendCollectedEvents(out, events)
- events = events[0:0]
- sendTimeSet = false
- }
- continue
- }
- out <- ev
- if ev.status == fileStatusReloadEnd {
- buffer = true
- }
- case now := <-ticker.C:
- if sendTimeSet && now.After(sendTime) {
- sendCollectedEvents(out, events)
- events = events[0:0]
- sendTimeSet = false
- }
- }
- }
-}
DELETED place/dirplace/directory/watch_test.go
Index: place/dirplace/directory/watch_test.go
==================================================================
--- place/dirplace/directory/watch_test.go
+++ /dev/null
@@ -1,57 +0,0 @@
-//-----------------------------------------------------------------------------
-// Copyright (c) 2020 Detlef Stern
-//
-// This file is part of zettelstore.
-//
-// Zettelstore is licensed under the latest version of the EUPL (European Union
-// Public License). Please see file LICENSE.txt for your rights and obligations
-// under this license.
-//-----------------------------------------------------------------------------
-
-// Package directory manages the directory part of a directory place.
-package directory
-
-import (
- "testing"
-)
-
-func sameStringSlices(sl1, sl2 []string) bool {
- if len(sl1) != len(sl2) {
- return false
- }
- for i := 0; i < len(sl1); i++ {
- if sl1[i] != sl2[i] {
- return false
- }
- }
- return true
-}
-
-func TestMatchValidFileName(t *testing.T) {
- testcases := []struct {
- name string
- exp []string
- }{
- {"", []string{}},
- {".txt", []string{}},
- {"12345678901234.txt", []string{"12345678901234", ".txt", "txt"}},
- {"12345678901234abc.txt", []string{"12345678901234", ".txt", "txt"}},
- {"12345678901234.abc.txt", []string{"12345678901234", ".txt", "txt"}},
- }
-
- for i, tc := range testcases {
- got := matchValidFileName(tc.name)
- if len(got) == 0 {
- if len(tc.exp) > 0 {
- t.Errorf("TC=%d, name=%q, exp=%v, got=%v", i, tc.name, tc.exp, got)
- }
- } else {
- if got[0] != tc.name {
- t.Errorf("TC=%d, name=%q, got=%v", i, tc.name, got)
- }
- if !sameStringSlices(got[1:], tc.exp) {
- t.Errorf("TC=%d, name=%q, exp=%v, got=%v", i, tc.name, tc.exp, got)
- }
- }
- }
-}
Index: place/dirplace/dirplace.go
==================================================================
--- place/dirplace/dirplace.go
+++ place/dirplace/dirplace.go
@@ -24,10 +24,11 @@
"zettelstore.de/z/config/runtime"
"zettelstore.de/z/domain"
"zettelstore.de/z/domain/id"
"zettelstore.de/z/domain/meta"
"zettelstore.de/z/place"
+ "zettelstore.de/z/place/change"
"zettelstore.de/z/place/dirplace/directory"
"zettelstore.de/z/place/fileplace"
"zettelstore.de/z/place/manager"
"zettelstore.de/z/search"
)
@@ -36,22 +37,32 @@
manager.Register("dir", func(u *url.URL, cdata *manager.ConnectData) (place.ManagedPlace, error) {
path := getDirPath(u)
if _, err := os.Stat(path); os.IsNotExist(err) {
return nil, err
}
+ dirSrvSpec, defWorker, maxWorker := getDirSrvInfo(u.Query().Get("type"))
dp := dirPlace{
- u: u,
- readonly: getQueryBool(u, "readonly"),
- cdata: *cdata,
- dir: path,
- dirRescan: time.Duration(
- getQueryInt(u, "rescan", 60, 3600, 30*24*60*60)) * time.Second,
- fSrvs: uint32(getQueryInt(u, "worker", 1, 17, 1499)),
+ location: u.String(),
+ readonly: getQueryBool(u, "readonly"),
+ cdata: *cdata,
+ dir: path,
+ dirRescan: time.Duration(getQueryInt(u, "rescan", 60, 3600, 30*24*60*60)) * time.Second,
+ dirSrvSpec: dirSrvSpec,
+ fSrvs: uint32(getQueryInt(u, "worker", 1, defWorker, maxWorker)),
}
return &dp, nil
})
}
+
+type directoryServiceSpec int
+
+const (
+ _ directoryServiceSpec = iota
+ dirSrvAny
+ dirSrvSimple
+ dirSrvNotify
+)
func getDirPath(u *url.URL) string {
if u.Opaque != "" {
return filepath.Clean(u.Opaque)
}
@@ -81,23 +92,25 @@
return iVal
}
// dirPlace uses a directory to store zettel as files.
type dirPlace struct {
- u *url.URL
- readonly bool
- cdata manager.ConnectData
- dir string
- dirRescan time.Duration
- dirSrv *directory.Service
- fSrvs uint32
- fCmds []chan fileCmd
- mxCmds sync.RWMutex
+ location string
+ readonly bool
+ cdata manager.ConnectData
+ dir string
+ dirRescan time.Duration
+ dirSrvSpec directoryServiceSpec
+ dirSrv directory.Service
+ mustNotify bool
+ fSrvs uint32
+ fCmds []chan fileCmd
+ mxCmds sync.RWMutex
}
func (dp *dirPlace) Location() string {
- return dp.u.String()
+ return dp.location
}
func (dp *dirPlace) Start(ctx context.Context) error {
dp.mxCmds.Lock()
dp.fCmds = make([]chan fileCmd, 0, dp.fSrvs)
@@ -104,14 +117,34 @@
for i := uint32(0); i < dp.fSrvs; i++ {
cc := make(chan fileCmd)
go fileService(i, cc)
dp.fCmds = append(dp.fCmds, cc)
}
- dp.dirSrv = directory.NewService(dp.dir, dp.dirRescan, dp.cdata.Notify)
+ dp.setupDirService()
dp.mxCmds.Unlock()
- dp.dirSrv.Start()
- return nil
+ if dp.dirSrv == nil {
+ panic("No directory service")
+ }
+ return dp.dirSrv.Start()
+}
+
+func (dp *dirPlace) Stop(ctx context.Context) error {
+ dirSrv := dp.dirSrv
+ dp.dirSrv = nil
+ err := dirSrv.Stop()
+ for _, c := range dp.fCmds {
+ close(c)
+ }
+ return err
+}
+
+func (dp *dirPlace) notifyChanged(reason change.Reason, zid id.Zid) {
+ if dp.mustNotify {
+ if chci := dp.cdata.Notify; chci != nil {
+ chci <- change.Info{Reason: reason, Zid: zid}
+ }
+ }
}
func (dp *dirPlace) getFileChan(zid id.Zid) chan fileCmd {
// Based on https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
var sum uint32 = 2166136261 ^ uint32(zid)
@@ -122,83 +155,83 @@
dp.mxCmds.RLock()
defer dp.mxCmds.RUnlock()
return dp.fCmds[sum%dp.fSrvs]
}
-func (dp *dirPlace) Stop(ctx context.Context) error {
- dirSrv := dp.dirSrv
- dp.dirSrv = nil
- dirSrv.Stop()
- for _, c := range dp.fCmds {
- close(c)
- }
- return nil
-}
-
func (dp *dirPlace) CanCreateZettel(ctx context.Context) bool {
return !dp.readonly
}
func (dp *dirPlace) CreateZettel(ctx context.Context, zettel domain.Zettel) (id.Zid, error) {
if dp.readonly {
return id.Invalid, place.ErrReadOnly
}
+ entry, err := dp.dirSrv.GetNew()
+ if err != nil {
+ return id.Invalid, err
+ }
meta := zettel.Meta
- entry := dp.dirSrv.GetNew()
meta.Zid = entry.Zid
- dp.updateEntryFromMeta(&entry, meta)
+ dp.updateEntryFromMeta(entry, meta)
- err := setZettel(dp, &entry, zettel)
+ err = setZettel(dp, entry, zettel)
if err == nil {
- dp.dirSrv.UpdateEntry(&entry)
+ dp.dirSrv.UpdateEntry(entry)
}
+ dp.notifyChanged(change.OnUpdate, meta.Zid)
return meta.Zid, err
}
func (dp *dirPlace) GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error) {
- entry := dp.dirSrv.GetEntry(zid)
- if !entry.IsValid() {
+ entry, err := dp.dirSrv.GetEntry(zid)
+ if err != nil || !entry.IsValid() {
return domain.Zettel{}, place.ErrNotFound
}
- m, c, err := getMetaContent(dp, &entry, zid)
+ m, c, err := getMetaContent(dp, entry, zid)
if err != nil {
return domain.Zettel{}, err
}
dp.cleanupMeta(ctx, m)
zettel := domain.Zettel{Meta: m, Content: domain.NewContent(c)}
return zettel, nil
}
func (dp *dirPlace) GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) {
- entry := dp.dirSrv.GetEntry(zid)
- if !entry.IsValid() {
+ entry, err := dp.dirSrv.GetEntry(zid)
+ if err != nil || !entry.IsValid() {
return nil, place.ErrNotFound
}
- m, err := getMeta(dp, &entry, zid)
+ m, err := getMeta(dp, entry, zid)
if err != nil {
return nil, err
}
dp.cleanupMeta(ctx, m)
return m, nil
}
func (dp *dirPlace) FetchZids(ctx context.Context) (id.Set, error) {
- entries := dp.dirSrv.GetEntries()
+ entries, err := dp.dirSrv.GetEntries()
+ if err != nil {
+ return nil, err
+ }
result := id.NewSetCap(len(entries))
for _, entry := range entries {
result[entry.Zid] = true
}
return result, nil
}
func (dp *dirPlace) SelectMeta(ctx context.Context, match search.MetaMatchFunc) (res []*meta.Meta, err error) {
- entries := dp.dirSrv.GetEntries()
+ entries, err := dp.dirSrv.GetEntries()
+ if err != nil {
+ return nil, err
+ }
res = make([]*meta.Meta, 0, len(entries))
// The following loop could be parallelized if needed for performance.
for _, entry := range entries {
- m, err1 := getMeta(dp, &entry, entry.Zid)
+ m, err1 := getMeta(dp, entry, entry.Zid)
err = err1
if err != nil {
continue
}
dp.cleanupMeta(ctx, m)
@@ -225,23 +258,30 @@
meta := zettel.Meta
if !meta.Zid.IsValid() {
return &place.ErrInvalidID{Zid: meta.Zid}
}
- entry := dp.dirSrv.GetEntry(meta.Zid)
+ entry, err := dp.dirSrv.GetEntry(meta.Zid)
+ if err != nil {
+ return err
+ }
if !entry.IsValid() {
// Existing zettel, but new in this place.
- entry.Zid = meta.Zid
- dp.updateEntryFromMeta(&entry, meta)
+ entry = &directory.Entry{Zid: meta.Zid}
+ dp.updateEntryFromMeta(entry, meta)
} else if entry.MetaSpec == directory.MetaSpecNone {
defaultMeta := fileplace.CalcDefaultMeta(entry.Zid, entry.ContentExt)
if !meta.Equal(defaultMeta, true) {
- dp.updateEntryFromMeta(&entry, meta)
- dp.dirSrv.UpdateEntry(&entry)
+ dp.updateEntryFromMeta(entry, meta)
+ dp.dirSrv.UpdateEntry(entry)
}
}
- return setZettel(dp, &entry, zettel)
+ err = setZettel(dp, entry, zettel)
+ if err == nil {
+ dp.notifyChanged(change.OnUpdate, meta.Zid)
+ }
+ return err
}
func (dp *dirPlace) updateEntryFromMeta(entry *directory.Entry, meta *meta.Meta) {
entry.MetaSpec, entry.ContentExt = calcSpecExt(meta)
basePath := filepath.Join(dp.dir, entry.Zid.String())
@@ -278,21 +318,21 @@
return place.ErrReadOnly
}
if curZid == newZid {
return nil
}
- curEntry := dp.dirSrv.GetEntry(curZid)
- if !curEntry.IsValid() {
+ curEntry, err := dp.dirSrv.GetEntry(curZid)
+ if err != nil || !curEntry.IsValid() {
return place.ErrNotFound
}
// Check whether zettel with new ID already exists in this place
- if _, err := dp.GetMeta(ctx, newZid); err == nil {
+ if _, err = dp.GetMeta(ctx, newZid); err == nil {
return &place.ErrInvalidID{Zid: newZid}
}
- oldMeta, oldContent, err := getMetaContent(dp, &curEntry, curZid)
+ oldMeta, oldContent, err := getMetaContent(dp, curEntry, curZid)
if err != nil {
return err
}
newEntry := directory.Entry{
@@ -301,48 +341,56 @@
MetaPath: renamePath(curEntry.MetaPath, curZid, newZid),
ContentPath: renamePath(curEntry.ContentPath, curZid, newZid),
ContentExt: curEntry.ContentExt,
}
- if err := dp.dirSrv.RenameEntry(&curEntry, &newEntry); err != nil {
+ if err = dp.dirSrv.RenameEntry(curEntry, &newEntry); err != nil {
return err
}
oldMeta.Zid = newZid
newZettel := domain.Zettel{Meta: oldMeta, Content: domain.NewContent(oldContent)}
- if err := setZettel(dp, &newEntry, newZettel); err != nil {
+ if err = setZettel(dp, &newEntry, newZettel); err != nil {
// "Rollback" rename. No error checking...
- dp.dirSrv.RenameEntry(&newEntry, &curEntry)
+ dp.dirSrv.RenameEntry(&newEntry, curEntry)
return err
}
- return deleteZettel(dp, &curEntry, curZid)
+ err = deleteZettel(dp, curEntry, curZid)
+ if err == nil {
+ dp.notifyChanged(change.OnDelete, curZid)
+ dp.notifyChanged(change.OnUpdate, newZid)
+ }
+ return err
}
func (dp *dirPlace) CanDeleteZettel(ctx context.Context, zid id.Zid) bool {
if dp.readonly {
return false
}
- entry := dp.dirSrv.GetEntry(zid)
- return entry.IsValid()
+ entry, err := dp.dirSrv.GetEntry(zid)
+ return err == nil && entry.IsValid()
}
func (dp *dirPlace) DeleteZettel(ctx context.Context, zid id.Zid) error {
if dp.readonly {
return place.ErrReadOnly
}
- entry := dp.dirSrv.GetEntry(zid)
- if !entry.IsValid() {
+ entry, err := dp.dirSrv.GetEntry(zid)
+ if err != nil || !entry.IsValid() {
return nil
}
dp.dirSrv.DeleteEntry(zid)
- err := deleteZettel(dp, &entry, zid)
+ err = deleteZettel(dp, entry, zid)
+ if err == nil {
+ dp.notifyChanged(change.OnDelete, zid)
+ }
return err
}
func (dp *dirPlace) ReadStats(st *place.Stats) {
st.ReadOnly = dp.readonly
- st.Zettel = dp.dirSrv.NumEntries()
+ st.Zettel, _ = dp.dirSrv.NumEntries()
}
func (dp *dirPlace) cleanupMeta(ctx context.Context, m *meta.Meta) {
if role, ok := m.Get(meta.KeyRole); !ok || role == "" {
m.Set(meta.KeyRole, runtime.GetDefaultRole())
ADDED place/dirplace/makedir.go
Index: place/dirplace/makedir.go
==================================================================
--- /dev/null
+++ place/dirplace/makedir.go
@@ -0,0 +1,43 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2021 Detlef Stern
+//
+// This file is part of zettelstore.
+//
+// Zettelstore is licensed under the latest version of the EUPL (European Union
+// Public License). Please see file LICENSE.txt for your rights and obligations
+// under this license.
+//-----------------------------------------------------------------------------
+
+// Package dirplace provides a directory-based zettel place.
+package dirplace
+
+import (
+ "zettelstore.de/z/config/startup"
+ "zettelstore.de/z/place/dirplace/notifydir"
+ "zettelstore.de/z/place/dirplace/simpledir"
+)
+
+func getDirSrvInfo(dirType string) (directoryServiceSpec, int, int) {
+ for count := 0; count < 2; count++ {
+ switch dirType {
+ case startup.ValueDirPlaceTypeNotify:
+ return dirSrvNotify, 7, 1499
+ case startup.ValueDirPlaceTypeSimple:
+ return dirSrvSimple, 1, 1
+ default:
+ dirType = startup.DefaultDirPlaceType()
+ }
+ }
+ panic("unable to set default dir place type: " + dirType)
+}
+
+func (dp *dirPlace) setupDirService() {
+ switch dp.dirSrvSpec {
+ case dirSrvSimple:
+ dp.dirSrv = simpledir.NewService(dp.dir)
+ dp.mustNotify = true
+ default:
+ dp.dirSrv = notifydir.NewService(dp.dir, dp.dirRescan, dp.cdata.Notify)
+ dp.mustNotify = false
+ }
+}
ADDED place/dirplace/notifydir/notifydir.go
Index: place/dirplace/notifydir/notifydir.go
==================================================================
--- /dev/null
+++ place/dirplace/notifydir/notifydir.go
@@ -0,0 +1,126 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2020-2021 Detlef Stern
+//
+// This file is part of zettelstore.
+//
+// Zettelstore is licensed under the latest version of the EUPL (European Union
+// Public License). Please see file LICENSE.txt for your rights and obligations
+// under this license.
+//-----------------------------------------------------------------------------
+
+// Package notifydir manages the notified directory part of a dirstore.
+package notifydir
+
+import (
+ "time"
+
+ "zettelstore.de/z/domain/id"
+ "zettelstore.de/z/place/change"
+ "zettelstore.de/z/place/dirplace/directory"
+)
+
+// notifyService specifies a directory scan service.
+type notifyService struct {
+ dirPath string
+ rescanTime time.Duration
+ done chan struct{}
+ cmds chan dirCmd
+ infos chan<- change.Info
+}
+
+// NewService creates a new directory service.
+func NewService(directoryPath string, rescanTime time.Duration, chci chan<- change.Info) directory.Service {
+ srv := ¬ifyService{
+ dirPath: directoryPath,
+ rescanTime: rescanTime,
+ cmds: make(chan dirCmd),
+ infos: chci,
+ }
+ return srv
+}
+
+// Start makes the directory service operational.
+func (srv *notifyService) Start() error {
+ tick := make(chan struct{})
+ rawEvents := make(chan *fileEvent)
+ events := make(chan *fileEvent)
+
+ ready := make(chan int)
+ go srv.directoryService(events, ready)
+ go collectEvents(events, rawEvents)
+ go watchDirectory(srv.dirPath, rawEvents, tick)
+
+ if srv.done != nil {
+ panic("src.done already set")
+ }
+ srv.done = make(chan struct{})
+ go ping(tick, srv.rescanTime, srv.done)
+ <-ready
+ return nil
+}
+
+// Stop stops the directory service.
+func (srv *notifyService) Stop() error {
+ close(srv.done)
+ srv.done = nil
+ return nil
+}
+
+func (srv *notifyService) notifyChange(reason change.Reason, zid id.Zid) {
+ if chci := srv.infos; chci != nil {
+ chci <- change.Info{Reason: reason, Zid: zid}
+ }
+}
+
+// NumEntries returns the number of managed zettel.
+func (srv *notifyService) NumEntries() (int, error) {
+ resChan := make(chan resNumEntries)
+ srv.cmds <- &cmdNumEntries{resChan}
+ return <-resChan, nil
+}
+
+// GetEntries returns an unsorted list of all current directory entries.
+func (srv *notifyService) GetEntries() ([]*directory.Entry, error) {
+ resChan := make(chan resGetEntries)
+ srv.cmds <- &cmdGetEntries{resChan}
+ return <-resChan, nil
+}
+
+// GetEntry returns the entry with the specified zettel id. If there is no such
+// zettel id, an empty entry is returned.
+func (srv *notifyService) GetEntry(zid id.Zid) (*directory.Entry, error) {
+ resChan := make(chan resGetEntry)
+ srv.cmds <- &cmdGetEntry{zid, resChan}
+ return <-resChan, nil
+}
+
+// GetNew returns an entry with a new zettel id.
+func (srv *notifyService) GetNew() (*directory.Entry, error) {
+ resChan := make(chan resNewEntry)
+ srv.cmds <- &cmdNewEntry{resChan}
+ result := <-resChan
+ return result.entry, result.err
+}
+
+// UpdateEntry notifies the directory of an updated entry.
+func (srv *notifyService) UpdateEntry(entry *directory.Entry) error {
+ resChan := make(chan struct{})
+ srv.cmds <- &cmdUpdateEntry{entry, resChan}
+ <-resChan
+ return nil
+}
+
+// RenameEntry notifies the directory of an renamed entry.
+func (srv *notifyService) RenameEntry(curEntry, newEntry *directory.Entry) error {
+ resChan := make(chan resRenameEntry)
+ srv.cmds <- &cmdRenameEntry{curEntry, newEntry, resChan}
+ return <-resChan
+}
+
+// DeleteEntry removes a zettel id from the directory of entries.
+func (srv *notifyService) DeleteEntry(zid id.Zid) error {
+ resChan := make(chan struct{})
+ srv.cmds <- &cmdDeleteEntry{zid, resChan}
+ <-resChan
+ return nil
+}
ADDED place/dirplace/notifydir/service.go
Index: place/dirplace/notifydir/service.go
==================================================================
--- /dev/null
+++ place/dirplace/notifydir/service.go
@@ -0,0 +1,256 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2020-2021 Detlef Stern
+//
+// This file is part of zettelstore.
+//
+// Zettelstore is licensed under the latest version of the EUPL (European Union
+// Public License). Please see file LICENSE.txt for your rights and obligations
+// under this license.
+//-----------------------------------------------------------------------------
+
+// Package notifydir manages the notified directory part of a dirstore.
+package notifydir
+
+import (
+ "log"
+ "time"
+
+ "zettelstore.de/z/domain/id"
+ "zettelstore.de/z/place"
+ "zettelstore.de/z/place/change"
+ "zettelstore.de/z/place/dirplace/directory"
+)
+
+// ping sends every tick a signal to reload the directory list
+func ping(tick chan<- struct{}, rescanTime time.Duration, done <-chan struct{}) {
+ ticker := time.NewTicker(rescanTime)
+ defer close(tick)
+ for {
+ select {
+ case _, ok := <-ticker.C:
+ if !ok {
+ return
+ }
+ tick <- struct{}{}
+ case _, ok := <-done:
+ if !ok {
+ ticker.Stop()
+ return
+ }
+ }
+ }
+}
+
+func newEntry(ev *fileEvent) *directory.Entry {
+ de := new(directory.Entry)
+ de.Zid = ev.zid
+ updateEntry(de, ev)
+ return de
+}
+
+func updateEntry(de *directory.Entry, ev *fileEvent) {
+ if ev.ext == "meta" {
+ de.MetaSpec = directory.MetaSpecFile
+ de.MetaPath = ev.path
+ return
+ }
+ if de.ContentExt != "" && de.ContentExt != ev.ext {
+ de.Duplicates = true
+ return
+ }
+ if de.MetaSpec != directory.MetaSpecFile {
+ if ev.ext == "zettel" {
+ de.MetaSpec = directory.MetaSpecHeader
+ } else {
+ de.MetaSpec = directory.MetaSpecNone
+ }
+ }
+ de.ContentPath = ev.path
+ de.ContentExt = ev.ext
+}
+
+type dirMap map[id.Zid]*directory.Entry
+
+func dirMapUpdate(dm dirMap, ev *fileEvent) {
+ de := dm[ev.zid]
+ if de == nil {
+ dm[ev.zid] = newEntry(ev)
+ return
+ }
+ updateEntry(de, ev)
+}
+
+func deleteFromMap(dm dirMap, ev *fileEvent) {
+ if ev.ext == "meta" {
+ if entry, ok := dm[ev.zid]; ok {
+ if entry.MetaSpec == directory.MetaSpecFile {
+ entry.MetaSpec = directory.MetaSpecNone
+ return
+ }
+ }
+ }
+ delete(dm, ev.zid)
+}
+
+// directoryService is the main service.
+func (srv *notifyService) directoryService(events <-chan *fileEvent, ready chan<- int) {
+ curMap := make(dirMap)
+ var newMap dirMap
+ for {
+ select {
+ case ev, ok := <-events:
+ if !ok {
+ return
+ }
+ switch ev.status {
+ case fileStatusReloadStart:
+ newMap = make(dirMap)
+ case fileStatusReloadEnd:
+ curMap = newMap
+ newMap = nil
+ if ready != nil {
+ ready <- len(curMap)
+ close(ready)
+ ready = nil
+ }
+ srv.notifyChange(change.OnReload, id.Invalid)
+ case fileStatusError:
+ log.Println("DIRPLACE", "ERROR", ev.err)
+ case fileStatusUpdate:
+ srv.processFileUpdateEvent(ev, curMap, newMap)
+ case fileStatusDelete:
+ srv.processFileDeleteEvent(ev, curMap, newMap)
+ }
+ case cmd, ok := <-srv.cmds:
+ if ok {
+ cmd.run(curMap)
+ }
+ }
+ }
+}
+
+func (srv *notifyService) processFileUpdateEvent(ev *fileEvent, curMap, newMap dirMap) {
+ if newMap != nil {
+ dirMapUpdate(newMap, ev)
+ } else {
+ dirMapUpdate(curMap, ev)
+ srv.notifyChange(change.OnUpdate, ev.zid)
+ }
+}
+
+func (srv *notifyService) processFileDeleteEvent(ev *fileEvent, curMap, newMap dirMap) {
+ if newMap != nil {
+ deleteFromMap(newMap, ev)
+ } else {
+ deleteFromMap(curMap, ev)
+ srv.notifyChange(change.OnDelete, ev.zid)
+ }
+}
+
+type dirCmd interface {
+ run(m dirMap)
+}
+
+type cmdNumEntries struct {
+ result chan<- resNumEntries
+}
+type resNumEntries = int
+
+func (cmd *cmdNumEntries) run(m dirMap) {
+ cmd.result <- len(m)
+}
+
+type cmdGetEntries struct {
+ result chan<- resGetEntries
+}
+type resGetEntries []*directory.Entry
+
+func (cmd *cmdGetEntries) run(m dirMap) {
+ res := make([]*directory.Entry, len(m))
+ i := 0
+ for _, de := range m {
+ entry := *de
+ res[i] = &entry
+ i++
+ }
+ cmd.result <- res
+}
+
+type cmdGetEntry struct {
+ zid id.Zid
+ result chan<- resGetEntry
+}
+type resGetEntry = *directory.Entry
+
+func (cmd *cmdGetEntry) run(m dirMap) {
+ entry := m[cmd.zid]
+ if entry == nil {
+ cmd.result <- nil
+ } else {
+ result := *entry
+ cmd.result <- &result
+ }
+}
+
+type cmdNewEntry struct {
+ result chan<- resNewEntry
+}
+type resNewEntry struct {
+ entry *directory.Entry
+ err error
+}
+
+func (cmd *cmdNewEntry) run(m dirMap) {
+ zid, err := place.GetNewZid(func(zid id.Zid) (bool, error) {
+ _, ok := m[zid]
+ return !ok, nil
+ })
+ if err != nil {
+ cmd.result <- resNewEntry{nil, err}
+ return
+ }
+ entry := &directory.Entry{Zid: zid}
+ m[zid] = entry
+ cmd.result <- resNewEntry{&directory.Entry{Zid: zid}, nil}
+}
+
+type cmdUpdateEntry struct {
+ entry *directory.Entry
+ result chan<- struct{}
+}
+
+func (cmd *cmdUpdateEntry) run(m dirMap) {
+ entry := *cmd.entry
+ m[entry.Zid] = &entry
+ cmd.result <- struct{}{}
+}
+
+type cmdRenameEntry struct {
+ curEntry *directory.Entry
+ newEntry *directory.Entry
+ result chan<- resRenameEntry
+}
+
+type resRenameEntry = error
+
+func (cmd *cmdRenameEntry) run(m dirMap) {
+ newEntry := *cmd.newEntry
+ newZid := newEntry.Zid
+ if _, found := m[newZid]; found {
+ cmd.result <- &place.ErrInvalidID{Zid: newZid}
+ return
+ }
+ delete(m, cmd.curEntry.Zid)
+ m[newZid] = &newEntry
+ cmd.result <- nil
+}
+
+type cmdDeleteEntry struct {
+ zid id.Zid
+ result chan<- struct{}
+}
+
+func (cmd *cmdDeleteEntry) run(m dirMap) {
+ delete(m, cmd.zid)
+ cmd.result <- struct{}{}
+}
ADDED place/dirplace/notifydir/watch.go
Index: place/dirplace/notifydir/watch.go
==================================================================
--- /dev/null
+++ place/dirplace/notifydir/watch.go
@@ -0,0 +1,300 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2020-2021 Detlef Stern
+//
+// This file is part of zettelstore.
+//
+// Zettelstore is licensed under the latest version of the EUPL (European Union
+// Public License). Please see file LICENSE.txt for your rights and obligations
+// under this license.
+//-----------------------------------------------------------------------------
+
+// Package notifydir manages the notified directory part of a dirstore.
+package notifydir
+
+import (
+ "os"
+ "path/filepath"
+ "regexp"
+ "time"
+
+ "github.com/fsnotify/fsnotify"
+
+ "zettelstore.de/z/domain/id"
+)
+
+var validFileName = regexp.MustCompile(`^(\d{14}).*(\.(.+))$`)
+
+func matchValidFileName(name string) []string {
+ return validFileName.FindStringSubmatch(name)
+}
+
+type fileStatus int
+
+const (
+ fileStatusNone fileStatus = iota
+ fileStatusReloadStart
+ fileStatusReloadEnd
+ fileStatusError
+ fileStatusUpdate
+ fileStatusDelete
+)
+
+type fileEvent struct {
+ status fileStatus
+ path string // Full file path
+ zid id.Zid
+ ext string // File extension
+ err error // Error if Status == fileStatusError
+}
+
+type sendResult int
+
+const (
+ sendDone sendResult = iota
+ sendReload
+ sendExit
+)
+
+func watchDirectory(directory string, events chan<- *fileEvent, tick <-chan struct{}) {
+ defer close(events)
+
+ var watcher *fsnotify.Watcher
+ defer func() {
+ if watcher != nil {
+ watcher.Close()
+ }
+ }()
+
+ sendEvent := func(ev *fileEvent) sendResult {
+ select {
+ case events <- ev:
+ case _, ok := <-tick:
+ if ok {
+ return sendReload
+ }
+ return sendExit
+ }
+ return sendDone
+ }
+
+ sendError := func(err error) sendResult {
+ return sendEvent(&fileEvent{status: fileStatusError, err: err})
+ }
+
+ sendFileEvent := func(status fileStatus, path string, match []string) sendResult {
+ zid, err := id.Parse(match[1])
+ if err != nil {
+ return sendDone
+ }
+ event := &fileEvent{
+ status: status,
+ path: path,
+ zid: zid,
+ ext: match[3],
+ }
+ return sendEvent(event)
+ }
+
+ reloadStartEvent := &fileEvent{status: fileStatusReloadStart}
+ reloadEndEvent := &fileEvent{status: fileStatusReloadEnd}
+ reloadFiles := func() bool {
+ entries, err := os.ReadDir(directory)
+ if err != nil {
+ if res := sendError(err); res != sendDone {
+ return res == sendReload
+ }
+ return true
+ }
+
+ if res := sendEvent(reloadStartEvent); res != sendDone {
+ return res == sendReload
+ }
+
+ if watcher != nil {
+ watcher.Close()
+ }
+ watcher, err = fsnotify.NewWatcher()
+ if err != nil {
+ if res := sendError(err); res != sendDone {
+ return res == sendReload
+ }
+ }
+
+ for _, entry := range entries {
+ if entry.IsDir() {
+ continue
+ }
+ if info, err1 := entry.Info(); err1 != nil || !info.Mode().IsRegular() {
+ continue
+ }
+ name := entry.Name()
+ match := matchValidFileName(name)
+ if len(match) > 0 {
+ path := filepath.Join(directory, name)
+ if res := sendFileEvent(fileStatusUpdate, path, match); res != sendDone {
+ return res == sendReload
+ }
+ }
+ }
+
+ if watcher != nil {
+ err = watcher.Add(directory)
+ if err != nil {
+ if res := sendError(err); res != sendDone {
+ return res == sendReload
+ }
+ }
+ }
+ if res := sendEvent(reloadEndEvent); res != sendDone {
+ return res == sendReload
+ }
+ return true
+ }
+
+ handleEvents := func() bool {
+ const createOps = fsnotify.Create | fsnotify.Write
+ const deleteOps = fsnotify.Remove | fsnotify.Rename
+
+ for {
+ select {
+ case wevent, ok := <-watcher.Events:
+ if !ok {
+ return false
+ }
+ path := filepath.Clean(wevent.Name)
+ match := matchValidFileName(filepath.Base(path))
+ if len(match) == 0 {
+ continue
+ }
+ if wevent.Op&createOps != 0 {
+ if fi, err := os.Lstat(path); err != nil || !fi.Mode().IsRegular() {
+ continue
+ }
+ if res := sendFileEvent(
+ fileStatusUpdate, path, match); res != sendDone {
+ return res == sendReload
+ }
+ }
+ if wevent.Op&deleteOps != 0 {
+ if res := sendFileEvent(
+ fileStatusDelete, path, match); res != sendDone {
+ return res == sendReload
+ }
+ }
+ case err, ok := <-watcher.Errors:
+ if !ok {
+ return false
+ }
+ if res := sendError(err); res != sendDone {
+ return res == sendReload
+ }
+ case _, ok := <-tick:
+ return ok
+ }
+ }
+ }
+
+ for {
+ if !reloadFiles() {
+ return
+ }
+ if watcher == nil {
+ if _, ok := <-tick; !ok {
+ return
+ }
+ } else {
+ if !handleEvents() {
+ return
+ }
+ }
+ }
+}
+
+func sendCollectedEvents(out chan<- *fileEvent, events []*fileEvent) {
+ for _, ev := range events {
+ if ev.status != fileStatusNone {
+ out <- ev
+ }
+ }
+}
+
+func addEvent(events []*fileEvent, ev *fileEvent) []*fileEvent {
+ switch ev.status {
+ case fileStatusNone:
+ return events
+ case fileStatusReloadStart:
+ events = events[0:0]
+ case fileStatusUpdate, fileStatusDelete:
+ if len(events) > 0 && mergeEvents(events, ev) {
+ return events
+ }
+ }
+ return append(events, ev)
+}
+
+func mergeEvents(events []*fileEvent, ev *fileEvent) bool {
+ for i := len(events) - 1; i >= 0; i-- {
+ oev := events[i]
+ switch oev.status {
+ case fileStatusReloadStart, fileStatusReloadEnd:
+ return false
+ case fileStatusUpdate, fileStatusDelete:
+ if ev.path == oev.path {
+ if ev.status == oev.status {
+ return true
+ }
+ oev.status = fileStatusNone
+ return false
+ }
+ }
+ }
+ return false
+}
+
+func collectEvents(out chan<- *fileEvent, in <-chan *fileEvent) {
+ defer close(out)
+
+ var sendTime time.Time
+ sendTimeSet := false
+ ticker := time.NewTicker(500 * time.Millisecond)
+ defer ticker.Stop()
+
+ events := make([]*fileEvent, 0, 32)
+ buffer := false
+ for {
+ select {
+ case ev, ok := <-in:
+ if !ok {
+ sendCollectedEvents(out, events)
+ return
+ }
+ if ev.status == fileStatusReloadStart {
+ buffer = false
+ events = events[0:0]
+ }
+ if buffer {
+ if !sendTimeSet {
+ sendTime = time.Now().Add(1500 * time.Millisecond)
+ sendTimeSet = true
+ }
+ events = addEvent(events, ev)
+ if len(events) > 1024 {
+ sendCollectedEvents(out, events)
+ events = events[0:0]
+ sendTimeSet = false
+ }
+ continue
+ }
+ out <- ev
+ if ev.status == fileStatusReloadEnd {
+ buffer = true
+ }
+ case now := <-ticker.C:
+ if sendTimeSet && now.After(sendTime) {
+ sendCollectedEvents(out, events)
+ events = events[0:0]
+ sendTimeSet = false
+ }
+ }
+ }
+}
ADDED place/dirplace/notifydir/watch_test.go
Index: place/dirplace/notifydir/watch_test.go
==================================================================
--- /dev/null
+++ place/dirplace/notifydir/watch_test.go
@@ -0,0 +1,55 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2020 Detlef Stern
+//
+// This file is part of zettelstore.
+//
+// Zettelstore is licensed under the latest version of the EUPL (European Union
+// Public License). Please see file LICENSE.txt for your rights and obligations
+// under this license.
+//-----------------------------------------------------------------------------
+
+// Package notifydir manages the notified directory part of a dirstore.
+package notifydir
+
+import "testing"
+
+func sameStringSlices(sl1, sl2 []string) bool {
+ if len(sl1) != len(sl2) {
+ return false
+ }
+ for i := 0; i < len(sl1); i++ {
+ if sl1[i] != sl2[i] {
+ return false
+ }
+ }
+ return true
+}
+
+func TestMatchValidFileName(t *testing.T) {
+ testcases := []struct {
+ name string
+ exp []string
+ }{
+ {"", []string{}},
+ {".txt", []string{}},
+ {"12345678901234.txt", []string{"12345678901234", ".txt", "txt"}},
+ {"12345678901234abc.txt", []string{"12345678901234", ".txt", "txt"}},
+ {"12345678901234.abc.txt", []string{"12345678901234", ".txt", "txt"}},
+ }
+
+ for i, tc := range testcases {
+ got := matchValidFileName(tc.name)
+ if len(got) == 0 {
+ if len(tc.exp) > 0 {
+ t.Errorf("TC=%d, name=%q, exp=%v, got=%v", i, tc.name, tc.exp, got)
+ }
+ } else {
+ if got[0] != tc.name {
+ t.Errorf("TC=%d, name=%q, got=%v", i, tc.name, got)
+ }
+ if !sameStringSlices(got[1:], tc.exp) {
+ t.Errorf("TC=%d, name=%q, exp=%v, got=%v", i, tc.name, tc.exp, got)
+ }
+ }
+ }
+}
Index: place/dirplace/service.go
==================================================================
--- place/dirplace/service.go
+++ place/dirplace/service.go
@@ -145,11 +145,11 @@
err = cmd.runMetaSpecHeader()
case directory.MetaSpecNone:
// TODO: if meta has some additional infos: write meta to new .meta;
// update entry in dir
err = writeFileContent(cmd.entry.ContentPath, cmd.zettel.Content.AsString())
- case directory.MetaSpecUnknown:
+ default:
panic("TODO: ???")
}
cmd.rc <- err
}
@@ -217,11 +217,11 @@
}
case directory.MetaSpecHeader:
err = os.Remove(cmd.entry.ContentPath)
case directory.MetaSpecNone:
err = os.Remove(cmd.entry.ContentPath)
- case directory.MetaSpecUnknown:
+ default:
panic("TODO: ???")
}
cmd.rc <- err
}
ADDED place/dirplace/simpledir/simpledir.go
Index: place/dirplace/simpledir/simpledir.go
==================================================================
--- /dev/null
+++ place/dirplace/simpledir/simpledir.go
@@ -0,0 +1,185 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2021 Detlef Stern
+//
+// This file is part of zettelstore.
+//
+// Zettelstore is licensed under the latest version of the EUPL (European Union
+// Public License). Please see file LICENSE.txt for your rights and obligations
+// under this license.
+//-----------------------------------------------------------------------------
+
+// Package simpledir manages the directory part of a dirstore.
+package simpledir
+
+import (
+ "os"
+ "path/filepath"
+ "regexp"
+ "sync"
+
+ "zettelstore.de/z/domain/id"
+ "zettelstore.de/z/place"
+ "zettelstore.de/z/place/dirplace/directory"
+)
+
+// simpleService specifies a directory service without scanning.
+type simpleService struct {
+ dirPath string
+ mx sync.Mutex
+}
+
+// NewService creates a new directory service.
+func NewService(directoryPath string) directory.Service {
+ return &simpleService{
+ dirPath: directoryPath,
+ }
+}
+
+func (ss *simpleService) Start() error {
+ ss.mx.Lock()
+ defer ss.mx.Unlock()
+ _, err := os.ReadDir(ss.dirPath)
+ return err
+}
+
+func (ss *simpleService) Stop() error {
+ return nil
+}
+
+func (ss *simpleService) NumEntries() (int, error) {
+ ss.mx.Lock()
+ defer ss.mx.Unlock()
+ entries, err := ss.getEntries()
+ if err == nil {
+ return len(entries), nil
+ }
+ return 0, err
+}
+
+func (ss *simpleService) GetEntries() ([]*directory.Entry, error) {
+ ss.mx.Lock()
+ defer ss.mx.Unlock()
+ entrySet, err := ss.getEntries()
+ if err != nil {
+ return nil, err
+ }
+ result := make([]*directory.Entry, 0, len(entrySet))
+ for _, entry := range entrySet {
+ result = append(result, entry)
+ }
+ return result, nil
+}
+func (ss *simpleService) getEntries() (map[id.Zid]*directory.Entry, error) {
+ dirEntries, err := os.ReadDir(ss.dirPath)
+ if err != nil {
+ return nil, err
+ }
+ entrySet := make(map[id.Zid]*directory.Entry)
+ for _, dirEntry := range dirEntries {
+ if dirEntry.IsDir() {
+ continue
+ }
+ if info, err1 := dirEntry.Info(); err1 != nil || !info.Mode().IsRegular() {
+ continue
+ }
+ name := dirEntry.Name()
+ match := matchValidFileName(name)
+ if len(match) == 0 {
+ continue
+ }
+ zid, err := id.Parse(match[1])
+ if err != nil {
+ continue
+ }
+ var entry *directory.Entry
+ if e, ok := entrySet[zid]; ok {
+ entry = e
+ } else {
+ entry = &directory.Entry{Zid: zid}
+ entrySet[zid] = entry
+ }
+ updateEntry(entry, filepath.Join(ss.dirPath, name), match[3])
+ }
+ return entrySet, nil
+}
+
+var validFileName = regexp.MustCompile(`^(\d{14}).*(\.(.+))$`)
+
+func matchValidFileName(name string) []string {
+ return validFileName.FindStringSubmatch(name)
+}
+
+func updateEntry(entry *directory.Entry, path, ext string) {
+ if ext == "meta" {
+ entry.MetaSpec = directory.MetaSpecFile
+ entry.MetaPath = path
+ } else if entry.ContentExt != "" && entry.ContentExt != ext {
+ entry.Duplicates = true
+ } else {
+ if entry.MetaSpec != directory.MetaSpecFile {
+ if ext == "zettel" {
+ entry.MetaSpec = directory.MetaSpecHeader
+ } else {
+ entry.MetaSpec = directory.MetaSpecNone
+ }
+ }
+ entry.ContentPath = path
+ entry.ContentExt = ext
+ }
+}
+
+func (ss *simpleService) GetEntry(zid id.Zid) (*directory.Entry, error) {
+ ss.mx.Lock()
+ defer ss.mx.Unlock()
+ return ss.getEntry(zid)
+}
+func (ss *simpleService) getEntry(zid id.Zid) (*directory.Entry, error) {
+ pattern := filepath.Join(ss.dirPath, zid.String()) + "*.*"
+ paths, err := filepath.Glob(pattern)
+ if err != nil {
+ return nil, err
+ }
+ if len(paths) == 0 {
+ return nil, nil
+ }
+ entry := &directory.Entry{Zid: zid}
+ for _, path := range paths {
+ ext := filepath.Ext(path)
+ if len(ext) > 0 && ext[0] == '.' {
+ ext = ext[1:]
+ }
+ updateEntry(entry, path, ext)
+ }
+ return entry, nil
+}
+
+func (ss *simpleService) GetNew() (*directory.Entry, error) {
+ ss.mx.Lock()
+ defer ss.mx.Unlock()
+ zid, err := place.GetNewZid(func(zid id.Zid) (bool, error) {
+ entry, err := ss.getEntry(zid)
+ if err != nil {
+ return false, nil
+ }
+ return !entry.IsValid(), nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ return &directory.Entry{Zid: zid}, nil
+}
+
+func (ss *simpleService) UpdateEntry(entry *directory.Entry) error {
+ // Noting to to, since the actual file update is done by dirplace
+ return nil
+}
+
+func (ss *simpleService) RenameEntry(curEntry, newEntry *directory.Entry) error {
+ // Noting to to, since the actual file rename is done by dirplace
+ return nil
+}
+
+func (ss *simpleService) DeleteEntry(zid id.Zid) error {
+ // Noting to to, since the actual file delete is done by dirplace
+ return nil
+}
ADDED place/helper.go
Index: place/helper.go
==================================================================
--- /dev/null
+++ place/helper.go
@@ -0,0 +1,37 @@
+//-----------------------------------------------------------------------------
+// Copyright (c) 2021 Detlef Stern
+//
+// This file is part of zettelstore.
+//
+// Zettelstore is licensed under the latest version of the EUPL (European Union
+// Public License). Please see file LICENSE.txt for your rights and obligations
+// under this license.
+//-----------------------------------------------------------------------------
+
+// Package place provides a generic interface to zettel places.
+package place
+
+import (
+ "time"
+
+ "zettelstore.de/z/domain/id"
+)
+
+// GetNewZid calculates a new and unused zettel identifier, based on the current date and time.
+func GetNewZid(testZid func(id.Zid) (bool, error)) (id.Zid, error) {
+ withSeconds := false
+ for i := 0; i < 90; i++ { // Must be completed within 9 seconds (less than web/server.writeTimeout)
+ zid := id.New(withSeconds)
+ found, err := testZid(zid)
+ if err != nil {
+ return id.Invalid, err
+ }
+ if found {
+ return zid, nil
+ }
+ // TODO: do not wait here unconditionally.
+ time.Sleep(100 * time.Millisecond)
+ withSeconds = true
+ }
+ return id.Invalid, ErrTimeout
+}
Index: place/memplace/memplace.go
==================================================================
--- place/memplace/memplace.go
+++ place/memplace/memplace.go
@@ -13,11 +13,10 @@
import (
"context"
"net/url"
"sync"
- "time"
"zettelstore.de/z/domain"
"zettelstore.de/z/domain/id"
"zettelstore.de/z/domain/meta"
"zettelstore.de/z/place"
@@ -67,31 +66,25 @@
func (mp *memPlace) CanCreateZettel(ctx context.Context) bool { return true }
func (mp *memPlace) CreateZettel(ctx context.Context, zettel domain.Zettel) (id.Zid, error) {
mp.mx.Lock()
+ zid, err := place.GetNewZid(func(zid id.Zid) (bool, error) {
+ _, ok := mp.zettel[zid]
+ return !ok, nil
+ })
+ if err != nil {
+ mp.mx.Unlock()
+ return id.Invalid, err
+ }
meta := zettel.Meta.Clone()
- meta.Zid = mp.calcNewZid()
+ meta.Zid = zid
zettel.Meta = meta
- mp.zettel[meta.Zid] = zettel
+ mp.zettel[zid] = zettel
mp.mx.Unlock()
- mp.notifyChanged(change.OnUpdate, meta.Zid)
- return meta.Zid, nil
-}
-
-func (mp *memPlace) calcNewZid() id.Zid {
- zid := id.New(false)
- if _, ok := mp.zettel[zid]; !ok {
- return zid
- }
- for {
- zid = id.New(true)
- if _, ok := mp.zettel[zid]; !ok {
- return zid
- }
- time.Sleep(100 * time.Millisecond)
- }
+ mp.notifyChanged(change.OnUpdate, zid)
+ return zid, nil
}
func (mp *memPlace) GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error) {
mp.mx.RLock()
zettel, ok := mp.zettel[zid]
Index: place/place.go
==================================================================
--- place/place.go
+++ place/place.go
@@ -168,10 +168,14 @@
// ErrReadOnly is returned if there is an attepmt to write to a read-only place.
var ErrReadOnly = errors.New("read-only place")
// ErrNotFound is returned if a zettel was not found in the place.
var ErrNotFound = errors.New("zettel not found")
+
+// ErrTimeout is returned if a place operation takes too long.
+// One example: if calculating a new zettel identifier takes too long.
+var ErrTimeout = errors.New("timeout")
// ErrInvalidID is returned if the zettel id is not appropriate for the place operation.
type ErrInvalidID struct{ Zid id.Zid }
func (err *ErrInvalidID) Error() string { return "invalid Zettel id: " + err.Zid.String() }
Index: place/progplace/config.go
==================================================================
--- place/progplace/config.go
+++ place/progplace/config.go
@@ -79,7 +79,8 @@
fmt.Fprintf(&sb, "|Secure cookie|%v\n", startup.SecureCookie())
fmt.Fprintf(&sb, "|Persistent Cookie|%v\n", startup.PersistentCookie())
html, api := startup.TokenLifetime()
fmt.Fprintf(&sb, "|API Token lifetime|%v\n", api)
fmt.Fprintf(&sb, "|HTML Token lifetime|%v\n", html)
+ fmt.Fprintf(&sb, "|Default directory place type|%v", startup.DefaultDirPlaceType())
return sb.String()
}
Index: tests/regression_test.go
==================================================================
--- tests/regression_test.go
+++ tests/regression_test.go
@@ -13,16 +13,18 @@
import (
"context"
"fmt"
"io"
+ "net/url"
"os"
"path/filepath"
"strings"
"testing"
"zettelstore.de/z/ast"
+ "zettelstore.de/z/config/startup"
"zettelstore.de/z/domain"
"zettelstore.de/z/domain/meta"
"zettelstore.de/z/encoder"
"zettelstore.de/z/parser"
"zettelstore.de/z/place"
@@ -49,11 +51,11 @@
cdata := manager.ConnectData{Filter: &noFilter{}, Notify: nil}
for _, entry := range entries {
if entry.IsDir() {
place, err := manager.Connect(
- "dir://"+filepath.Join(root, entry.Name()),
+ "dir://"+filepath.Join(root, entry.Name())+"?type="+startup.ValueDirPlaceTypeSimple,
false,
&cdata,
)
if err != nil {
panic(err)
@@ -129,11 +131,15 @@
t.Errorf("\n1st: %q\n2nd: %q", gotFirst, gotSecond)
}
}
func getPlaceName(p place.ManagedPlace, root string) string {
- return p.Location()[len("dir://")+len(root):]
+ u, err := url.Parse(p.Location())
+ if err != nil {
+ panic("Unable to parse URL '" + p.Location() + "': " + err.Error())
+ }
+ return u.Path[len(root):]
}
func match(*meta.Meta) bool { return true }
func checkContentPlace(t *testing.T, p place.ManagedPlace, wd, placeName string) {
Index: web/adapter/response.go
==================================================================
--- web/adapter/response.go
+++ web/adapter/response.go
@@ -57,7 +57,10 @@
return http.StatusBadRequest, err1.Text
}
if err == place.ErrStopped {
return http.StatusInternalServerError, fmt.Sprintf("Zettelstore not operational: %v.", err)
}
+ if err == place.ErrTimeout {
+ return http.StatusLoopDetected, "Zettelstore operation took too long"
+ }
return http.StatusInternalServerError, err.Error()
}
Index: www/changes.wiki
==================================================================
--- www/changes.wiki
+++ www/changes.wiki
@@ -1,9 +1,28 @@
Change Log
+
+Changes for Version 0.0.13 (pending)
+
-Changes for Version 0.0.12 (pending)
+Changes for Version 0.0.12 (2021-04-16)
+ * Raise the per-process limit of open files on macOS to 1.048.576. This
+ allows most macOS users to use at least 500.000 zettel. That should be
+ enough for the near future.
+ (major)
+ * Mitigate the shortcomings of the macOS version by introducing types of
+ directory places. The original directory place type is now called "notify"
+ (the default value). There is a new type called "simple". This new type
+ does not notify Zettelstore when some of the underlying Zettel files
+ change.
+ (major)
+ * Add new startup configuration default-dir-place-type, which gives
+ the default value for specifying a directory place type. The default value
+ is “notify”. On macOS, the default value may be changed
+ “simple” if some errors occur while raising the per-process
+ limit of open files.
+ (minor)
Changes for Version 0.0.11 (2021-04-05)
* New place schema "file" allows to read zettel from a ZIP file.
A zettel collection can now be packaged and distributed easier.
@@ -34,14 +53,12 @@
just matches the prefix of only one of its tags.
(minor: api, webui)
* Many smaller bug fixes and inprovements, to the software and to the documentation.
A note for users of macOS: in the current release and with macOS's default values,
-a zettel directory place must not contain more than approx. 250 files. There are four options
+a zettel directory place must not contain more than approx. 250 files. There are three options
to mitigate this limitation temporarily:
- # You [https://zettelstore.de/manual/h/00001004010000|re-configure] your Zettelstore to use more
- than one directory place.
# You update the per-process limit of open files on macOS.
# You setup a virtualization environment to run Zettelstore on Linux or Windows.
# You wait for version 0.0.12 which addresses this issue.
Index: www/download.wiki
==================================================================
--- www/download.wiki
+++ www/download.wiki
@@ -7,17 +7,17 @@
* However, it is in use by the main developer since March 2020 without any damage.
* It may be useful for you. It is useful for me.
* Take a look at the [https://zettelstore.de/manual/|manual] to know how to start and use it.
ZIP-ped Executables
-Build: v0.0.11 (2021-04-05).
+Build: v0.0.12 (2021-04-16).
- * [/uv/zettelstore-0.0.11-linux-amd64.zip|Linux] (amd64)
- * [/uv/zettelstore-0.0.11-linux-arm.zip|Linux] (arm6, e.g. Raspberry Pi)
- * [/uv/zettelstore-0.0.11-windows-amd64.zip|Windows] (amd64)
- * [/uv/zettelstore-0.0.11-darwin-amd64.zip|macOS] (amd64)
- * [/uv/zettelstore-0.0.11-darwin-arm64.zip|macOS] (arm64, aka Apple silicon)
+ * [/uv/zettelstore-0.0.12-linux-amd64.zip|Linux] (amd64)
+ * [/uv/zettelstore-0.0.12-linux-arm.zip|Linux] (arm6, e.g. Raspberry Pi)
+ * [/uv/zettelstore-0.0.12-windows-amd64.zip|Windows] (amd64)
+ * [/uv/zettelstore-0.0.12-darwin-amd64.zip|macOS] (amd64)
+ * [/uv/zettelstore-0.0.12-darwin-arm64.zip|macOS] (arm64, aka Apple silicon)
Unzip the appropriate file, install and execute Zettelstore according to the manual.
Zettel for the manual
As a starter, you can download the zettel for the manual [/uv/manual-0.0.11.zip|here].
Index: www/index.wiki
==================================================================
--- www/index.wiki
+++ www/index.wiki
@@ -15,17 +15,17 @@
The software, including the manual, is licensed under the
[/file?name=LICENSE.txt&ci=trunk|European Union Public License 1.2 (or later)].
[https://twitter.com/zettelstore|Stay tuned]…
-Latest Release: 0.0.11 (2021-04-05)
+Latest Release: 0.0.12 (2021-04-16)
* [./download.wiki|Download]
- * [./changes.wiki#0_0_11|Change Summary]
- * [/timeline?p=version-0.0.11&bt=version-0.0.10&y=ci|Check-ins for version 0.0.11],
- [/vdiff?to=version-0.0.11&from=version-0.0.10|content diff]
- * [/timeline?df=version-0.0.11&y=ci|Check-ins derived from the 0.0.11 release],
- [/vdiff?from=version-0.0.11&to=trunk|content diff]
+ * [./changes.wiki#0_0_12|Change Summary]
+ * [/timeline?p=version-0.0.12&bt=version-0.0.11&y=ci|Check-ins for version 0.0.12],
+ [/vdiff?to=version-0.0.12&from=version-0.0.11|content diff]
+ * [/timeline?df=version-0.0.12&y=ci|Check-ins derived from the 0.0.12 release],
+ [/vdiff?from=version-0.0.12&to=trunk|content diff]
* [./plan.wiki|Limitations and planned Improvements]
* [/timeline?t=release|Timeline of all past releases]
Build instructions