Zettelstore

0.0.12

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// +build !darwin

package cmd

func raiseFdLimit() error { return nil }

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// +build darwin

package cmd

import (
	"log"
	"syscall"
)

const minFiles = 1048576

func raiseFdLimit() error {
	var rLimit syscall.Rlimit
	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
	if err != nil {
		return err
	}
	if rLimit.Cur >= minFiles {
		return nil
	}
	rLimit.Cur = minFiles
	if rLimit.Cur > rLimit.Max {
		rLimit.Cur = rLimit.Max
	}
	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
	if err != nil {
		return err
	}
	err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
	if err != nil {
		return err
	}
	if rLimit.Cur < minFiles {
		log.Printf("Make sure you have no more than %d files in all your places if you enabled notification\n", rLimit.Cur)
	}
	return nil
}

package cmd

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"zettelstore.de/z/config/runtime"
	"zettelstore.de/z/config/startup"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"

	return cfg
}

func setupOperations(cfg *meta.Meta, withPlaces, simple bool) error {
	var mgr place.Manager
	var idx index.Indexer
	if withPlaces {
		err := raiseFdLimit()
		if err != nil {
			log.Println("Raising some limitions did not work:", err)
			log.Println("Prepare to encounter errors. Most of them can be mitigated. See the manual for details")
			cfg.Set(startup.KeyDefaultDirPlaceType, startup.ValueDirPlaceTypeSimple)
		}
		startup.SetupStartupConfig(cfg)
		idx = indexer.New()
		filter := index.NewMetaFilter(idx)
		mgr, err = manager.New(getPlaces(cfg), cfg.GetBool(startup.KeyReadOnlyMode), filter)
		if err != nil {
			return err
		}
	} else {
		startup.SetupStartupConfig(cfg)
	}

	startup.SetupStartupService(mgr, idx, simple)




	if withPlaces {
		if err := mgr.Start(context.Background()); err != nil {
			fmt.Fprintln(os.Stderr, "Unable to start zettel place")
			return err
		}
		runtime.SetupConfiguration(mgr)
		progplace.Setup(cfg, mgr, idx)

	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/index"
	"zettelstore.de/z/place"
)

var config struct {
	// Set in SetupStartupConfig
	verbose             bool
	readonlyMode        bool
	urlPrefix           string
	listenAddress       string
	defaultDirPlaceType string
	owner               id.Zid
	withAuth            bool
	secret              []byte
	insecCookie         bool
	persistCookie       bool
	htmlLifetime        time.Duration
	apiLifetime         time.Duration

	// Set in SetupStartupService
	simple  bool // was started without run command
	manager place.Manager
	indexer index.Indexer
}

// Predefined keys for startup zettel
const (
	KeyDefaultDirPlaceType = "default-dir-place-type"
	KeyInsecureCookie      = "insecure-cookie"
	KeyListenAddress       = "listen-addr"
	KeyOwner               = "owner"
	KeyPersistentCookie    = "persistent-cookie"
	KeyPlaceOneURI         = "place-1-uri"
	KeyReadOnlyMode        = "read-only-mode"
	KeyTokenLifetimeHTML   = "token-lifetime-html"
	KeyTokenLifetimeAPI    = "token-lifetime-api"
	KeyURLPrefix           = "url-prefix"
	KeyVerbose             = "verbose"
)

// Important values for some keys.
const (
	ValueDirPlaceTypeNotify = "notify"
	ValueDirPlaceTypeSimple = "simple"
)

// SetupStartupConfig initializes the startup data with content of config file.
func SetupStartupConfig(cfg *meta.Meta) {
	if config.urlPrefix != "" {
		panic("startup.config already set")
	}

	config.verbose = cfg.GetBool(KeyVerbose)
	config.readonlyMode = cfg.GetBool(KeyReadOnlyMode)
	config.urlPrefix = cfg.GetDefault(KeyURLPrefix, "/")
	if prefix, ok := cfg.Get(KeyURLPrefix); ok &&
		len(prefix) > 0 && prefix[0] == '/' && prefix[len(prefix)-1] == '/' {
		config.urlPrefix = prefix
	} else {
		config.urlPrefix = "/"
	}
	if val, ok := cfg.Get(KeyListenAddress); ok {
		config.listenAddress = val // TODO: check for valid string
	} else {
		config.listenAddress = "127.0.0.1:23123"
	}
	if defaultType, ok := cfg.Get(KeyDefaultDirPlaceType); ok {
		switch defaultType {
		case ValueDirPlaceTypeNotify:
		case ValueDirPlaceTypeSimple:
		default:
			defaultType = ValueDirPlaceTypeNotify
		}
		config.defaultDirPlaceType = defaultType
	} else {
		config.defaultDirPlaceType = ValueDirPlaceTypeNotify
	}
	config.owner = id.Invalid
	if owner, ok := cfg.Get(KeyOwner); ok {
		if zid, err := id.Parse(owner); err == nil {
			config.owner = zid
			config.withAuth = true
		}
	}
	if config.withAuth {
		config.insecCookie = cfg.GetBool(KeyInsecureCookie)
		config.persistCookie = cfg.GetBool(KeyPersistentCookie)
		config.secret = calcSecret(cfg)
		config.htmlLifetime = getDuration(
			cfg, KeyTokenLifetimeHTML, 1*time.Hour, 1*time.Minute, 30*24*time.Hour)
		config.apiLifetime = getDuration(
			cfg, KeyTokenLifetimeAPI, 10*time.Minute, 0, 1*time.Hour)
	}
}

// SetupStartupService initializes the startup data with internal services.
func SetupStartupService(manager place.Manager, idx index.Indexer, simple bool) {
	if config.urlPrefix == "" {
		panic("startup.config not set")
	}
	config.simple = simple && !config.withAuth
	config.manager = manager
	config.indexer = idx

}

func calcSecret(cfg *meta.Meta) []byte {
	h := fnv.New128()
	if secret, ok := cfg.Get("secret"); ok {
		io.WriteString(h, secret)
	}

// URLPrefix returns the configured prefix to be used when providing URL to
// the service.
func URLPrefix() string { return config.urlPrefix }

// ListenAddress returns the string that specifies the the network card and the ip port
// where the server listens for requests
func ListenAddress() string { return config.listenAddress }

// DefaultDirPlaceType returns the default value for a directory place type.
func DefaultDirPlaceType() string { return config.defaultDirPlaceType }

// WithAuth returns true if user authentication is enabled.
func WithAuth() bool { return config.withAuth }

// SecureCookie returns whether the web app should set cookies to secure mode.
func SecureCookie() bool { return config.withAuth && !config.insecCookie }

Therefore only the owner of the computer on which Zettelstore runs can change this information.

The file for start-up configuration must be created via a text editor in advance.

The syntax of the configuration file is the same as for any zettel metadata.
The following keys are supported:

; [!default-dir-place-type]''default-dir-place-type''
: Specifies the default value for the (sub-) type of [[directory places|00001004011400#type]].
  Zettel are typically stored in such places.

  Default: ''notify''
; [!insecure-cookie]''insecure-cookie''
: Must be set to ''true'', if authentication is enabled and Zettelstore is not accessible not via HTTPS (but via HTTP).
  Otherwise web browser are free to ignore the authentication cookie.

  Default: ''false''
; [!listen-addr]''listen-addr''
: Configures the network address, where is zettel web service is listening for requests.

id: 00001004011400
title: Configure file directory places
role: manual
tags: #configuration #manual #zettelstore
syntax: zmk


Under certain circumstances, it is preferable to further configure a file directory place.
This is done by appending query parameters after the base place URI ''dir:\//DIR''.

The following parameters are supported:

|= Parameter:|Description|Default value:|
|type|(Sub-) Type of the directory service|(value of ''[[default-dir-place-type|00001004010000#default-dir-place-type]]'')
|rescan|Time (in seconds) after which the directory should be scanned fully|600
|worker|Number of worker that can access the directory in parallel|(depends on type)
|readonly|Allow only operations that do not change a zettel or create a new zettel|n/a

=== Type
On some operating systems, Zettelstore tries to detect changes to zettel files outside of Zettelstore's control[^This includes Linux, Windows, and macOS.].
On other operating systems, this may be not possible, due to technical limitations.
Automatic detection of external changes is also not possible, if zettel files are placed on an external service, such as a file server accessed via SMD/CIFS or NFS.

To cope with this uncertainty, Zettelstore provides various internal implementations of a directory place.
The default values should match the needs of different users, as explained in the [[installation part|00001003000000]] of this manual.
The following values are supported:

; simple
: Is not able to detect external changes.
  Works on all platforms.
  Is a little slower than other implementations (up to three times slower).
; notify
: Automatically detect external changes.
  Tries to optimize performance, at a little cost of main memory (RAM).

=== Rescan
When the parameter ''type'' is set to ""notify"", Zettelstore automatically detects changes to zettel files that originates from other software.
It is done on a ""best-effort"" basis.
Under certain circumstances it is possible that Zettelstore does not detect a change done by another software.

To cope with this unlikely, but still possible event, Zettelstore re-scans periodically the file directory.
The time interval is configured by the ''rescan'' parameter, e.g.
```
place-1-uri: dir:///home/zettel?rescan=300
```
This makes Zettelstore to re-scan the directory ''/home/zettel/'' every 300 seconds, i.e. 5 minutes.

For a Zettelstore with many zettel, re-scanning the directory may take a while, especially if it is stored on a remote computer (e.g. via CIFS/SMB or NFS).
In this case, you should adjust the parameter value.

Please note that a directory re-scan invalidates all internal data of a Zettelstore.
It might trigger a re-build of the backlink database (and other internal databases).
Therefore a large value if preferred.

This value is ignored for other directory place type, such as ""simple"".

=== Worker
Internally, Zettelstore parallels concurrent requests for a zettel or its metadata.
The number of parallel activities is configured by the ''worker'' parameter.


A computer contains a limited number of internal processing units (CPU).
Its number ranges from 1 to (currently) 128, e.g. in bigger server environments.
Zettelstore typically runs on a system with 1 to 8 CPUs.
Access to zettel file is ultimately managed by the underlying operating system.
Depending on the hardware and on the type of the directory place, only a limited number of parallel accesses are desirable.


On smaller hardware[^In comparison to a normal desktop or laptop computer], such as the [[Raspberry Zero|https://www.raspberrypi.org/products/raspberry-pi-zero/]], a smaller value might be appropriate.
Every worker needs some amount of main memory (RAM) and some amount of processing power.
On bigger hardware, with some fast file services, a bigger value could result in higher performance, if needed.

For a directory place of type ""notify"", the default value is: 7.
The directory place type ""simple"" limits the value to a maximum of 1, i.e. no concurrency is possible with this type of directory place.

For various reasons, the value should be a prime number, with a maximum value of 1499.

=== Readonly
Sometimes you may want to provide zettel from a file directory place, but you want to disallow any changes.
If you provide the query parameter ''readonly'' (with or without a corresponding value), the place will disallow any changes.
```
place-1-uri: dir:///home/zettel?readonly
```
If you put the whole Zettelstore in [[read-only|00001004010000]] [[mode|00001004051000]], all configured file directory places will be in read-only mode too, even if not explicitly configured.

module zettelstore.de/z

go 1.16

require (
	github.com/fsnotify/fsnotify v1.4.9
	github.com/pascaldekloe/jwt v1.10.0
	github.com/yuin/goldmark v1.3.5
	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
	golang.org/x/term v0.0.0-20201117132131-f5c789dd3221
	golang.org/x/text v0.3.6
)

github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/pascaldekloe/jwt v1.10.0 h1:ktcIUV4TPvh404R5dIBEnPCsSwj0sqi3/0+XafE5gJs=
github.com/pascaldekloe/jwt v1.10.0/go.mod h1:TKhllgThT7TOP5rGr2zMLKEDZRAgJfBbtKyVeRsNB9A=
github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=

//-----------------------------------------------------------------------------
// Copyright (c) 2020-2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package directory manages the directory interface of a dirstore.
package directory




import "zettelstore.de/z/domain/id"


// Service is the interface of a directory service.

type Service interface {






	Start() error


	Stop() error



	NumEntries() (int, error)

	GetEntries() ([]*Entry, error)

	GetEntry(zid id.Zid) (*Entry, error)

	GetNew() (*Entry, error)


	UpdateEntry(entry *Entry) error
	RenameEntry(curEntry, newEntry *Entry) error




	DeleteEntry(zid id.Zid) error


}




// MetaSpec defines all possibilities where meta data can be stored.



type MetaSpec int


// Constants for MetaSpec
const (
	_              MetaSpec = iota
	MetaSpecNone            // no meta information



	MetaSpecFile            // meta information is in meta file






	MetaSpecHeader          // meta information is in header
)






// Entry stores everything for a directory entry.
type Entry struct {
	Zid         id.Zid





	MetaSpec    MetaSpec // location of meta information
	MetaPath    string   // file path of meta information
	ContentPath string   // file path of zettel content





	ContentExt  string   // (normalized) file extension of zettel content





	Duplicates  bool     // multiple content files
}

// IsValid checks whether the entry is valid.

func (e *Entry) IsValid() bool {



	return e != nil && e.Zid.IsValid()





}

	"time"

	"zettelstore.de/z/config/runtime"
	"zettelstore.de/z/domain"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/place"
	"zettelstore.de/z/place/change"
	"zettelstore.de/z/place/dirplace/directory"
	"zettelstore.de/z/place/fileplace"
	"zettelstore.de/z/place/manager"
	"zettelstore.de/z/search"
)

func init() {
	manager.Register("dir", func(u *url.URL, cdata *manager.ConnectData) (place.ManagedPlace, error) {
		path := getDirPath(u)
		if _, err := os.Stat(path); os.IsNotExist(err) {
			return nil, err
		}
		dirSrvSpec, defWorker, maxWorker := getDirSrvInfo(u.Query().Get("type"))
		dp := dirPlace{
			location:   u.String(),
			readonly:   getQueryBool(u, "readonly"),
			cdata:      *cdata,
			dir:        path,

			dirRescan:  time.Duration(getQueryInt(u, "rescan", 60, 3600, 30*24*60*60)) * time.Second,
			dirSrvSpec: dirSrvSpec,
			fSrvs:      uint32(getQueryInt(u, "worker", 1, defWorker, maxWorker)),
		}
		return &dp, nil
	})
}

type directoryServiceSpec int

const (
	_ directoryServiceSpec = iota
	dirSrvAny
	dirSrvSimple
	dirSrvNotify
)

func getDirPath(u *url.URL) string {
	if u.Opaque != "" {
		return filepath.Clean(u.Opaque)
	}
	return filepath.Clean(u.Path)
}

		return max
	}
	return iVal
}

// dirPlace uses a directory to store zettel as files.
type dirPlace struct {
	location   string
	readonly   bool
	cdata      manager.ConnectData
	dir        string
	dirRescan  time.Duration
	dirSrvSpec directoryServiceSpec
	dirSrv     directory.Service
	mustNotify bool
	fSrvs      uint32
	fCmds      []chan fileCmd
	mxCmds     sync.RWMutex
}

func (dp *dirPlace) Location() string {
	return dp.location
}

func (dp *dirPlace) Start(ctx context.Context) error {
	dp.mxCmds.Lock()
	dp.fCmds = make([]chan fileCmd, 0, dp.fSrvs)
	for i := uint32(0); i < dp.fSrvs; i++ {
		cc := make(chan fileCmd)
		go fileService(i, cc)
		dp.fCmds = append(dp.fCmds, cc)
	}
	dp.setupDirService()
	dp.mxCmds.Unlock()
	if dp.dirSrv == nil {
		panic("No directory service")
	}
	return dp.dirSrv.Start()
}

func (dp *dirPlace) Stop(ctx context.Context) error {
	dirSrv := dp.dirSrv
	dp.dirSrv = nil
	err := dirSrv.Stop()
	for _, c := range dp.fCmds {
		close(c)
	}
	return err
}

func (dp *dirPlace) notifyChanged(reason change.Reason, zid id.Zid) {
	if dp.mustNotify {
		if chci := dp.cdata.Notify; chci != nil {
			chci <- change.Info{Reason: reason, Zid: zid}
		}
	}
}

func (dp *dirPlace) getFileChan(zid id.Zid) chan fileCmd {
	// Based on https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
	var sum uint32 = 2166136261 ^ uint32(zid)
	sum *= 16777619
	sum ^= uint32(zid >> 32)
	sum *= 16777619

	dp.mxCmds.RLock()
	defer dp.mxCmds.RUnlock()
	return dp.fCmds[sum%dp.fSrvs]
}











func (dp *dirPlace) CanCreateZettel(ctx context.Context) bool {
	return !dp.readonly
}

func (dp *dirPlace) CreateZettel(ctx context.Context, zettel domain.Zettel) (id.Zid, error) {
	if dp.readonly {
		return id.Invalid, place.ErrReadOnly
	}

	entry, err := dp.dirSrv.GetNew()
	if err != nil {
		return id.Invalid, err
	}
	meta := zettel.Meta

	meta.Zid = entry.Zid
	dp.updateEntryFromMeta(entry, meta)

	err = setZettel(dp, entry, zettel)
	if err == nil {
		dp.dirSrv.UpdateEntry(entry)
	}
	dp.notifyChanged(change.OnUpdate, meta.Zid)
	return meta.Zid, err
}

func (dp *dirPlace) GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error) {
	entry, err := dp.dirSrv.GetEntry(zid)
	if err != nil || !entry.IsValid() {
		return domain.Zettel{}, place.ErrNotFound
	}
	m, c, err := getMetaContent(dp, entry, zid)
	if err != nil {
		return domain.Zettel{}, err
	}
	dp.cleanupMeta(ctx, m)
	zettel := domain.Zettel{Meta: m, Content: domain.NewContent(c)}
	return zettel, nil
}

func (dp *dirPlace) GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) {
	entry, err := dp.dirSrv.GetEntry(zid)
	if err != nil || !entry.IsValid() {
		return nil, place.ErrNotFound
	}
	m, err := getMeta(dp, entry, zid)
	if err != nil {
		return nil, err
	}
	dp.cleanupMeta(ctx, m)
	return m, nil
}

func (dp *dirPlace) FetchZids(ctx context.Context) (id.Set, error) {
	entries, err := dp.dirSrv.GetEntries()
	if err != nil {
		return nil, err
	}
	result := id.NewSetCap(len(entries))
	for _, entry := range entries {
		result[entry.Zid] = true
	}
	return result, nil
}

func (dp *dirPlace) SelectMeta(ctx context.Context, match search.MetaMatchFunc) (res []*meta.Meta, err error) {
	entries, err := dp.dirSrv.GetEntries()
	if err != nil {
		return nil, err
	}
	res = make([]*meta.Meta, 0, len(entries))
	// The following loop could be parallelized if needed for performance.
	for _, entry := range entries {
		m, err1 := getMeta(dp, entry, entry.Zid)
		err = err1
		if err != nil {
			continue
		}
		dp.cleanupMeta(ctx, m)
		dp.cdata.Filter.Enrich(ctx, m)

		return place.ErrReadOnly
	}

	meta := zettel.Meta
	if !meta.Zid.IsValid() {
		return &place.ErrInvalidID{Zid: meta.Zid}
	}
	entry, err := dp.dirSrv.GetEntry(meta.Zid)
	if err != nil {
		return err
	}
	if !entry.IsValid() {
		// Existing zettel, but new in this place.
		entry = &directory.Entry{Zid: meta.Zid}
		dp.updateEntryFromMeta(entry, meta)
	} else if entry.MetaSpec == directory.MetaSpecNone {
		defaultMeta := fileplace.CalcDefaultMeta(entry.Zid, entry.ContentExt)
		if !meta.Equal(defaultMeta, true) {
			dp.updateEntryFromMeta(entry, meta)
			dp.dirSrv.UpdateEntry(entry)
		}
	}
	err = setZettel(dp, entry, zettel)
	if err == nil {
		dp.notifyChanged(change.OnUpdate, meta.Zid)
	}
	return err
}

func (dp *dirPlace) updateEntryFromMeta(entry *directory.Entry, meta *meta.Meta) {
	entry.MetaSpec, entry.ContentExt = calcSpecExt(meta)
	basePath := filepath.Join(dp.dir, entry.Zid.String())
	if entry.MetaSpec == directory.MetaSpecFile {
		entry.MetaPath = basePath + ".meta"

func (dp *dirPlace) RenameZettel(ctx context.Context, curZid, newZid id.Zid) error {
	if dp.readonly {
		return place.ErrReadOnly
	}
	if curZid == newZid {
		return nil
	}
	curEntry, err := dp.dirSrv.GetEntry(curZid)
	if err != nil || !curEntry.IsValid() {
		return place.ErrNotFound
	}

	// Check whether zettel with new ID already exists in this place
	if _, err = dp.GetMeta(ctx, newZid); err == nil {
		return &place.ErrInvalidID{Zid: newZid}
	}

	oldMeta, oldContent, err := getMetaContent(dp, curEntry, curZid)
	if err != nil {
		return err
	}

	newEntry := directory.Entry{
		Zid:         newZid,
		MetaSpec:    curEntry.MetaSpec,
		MetaPath:    renamePath(curEntry.MetaPath, curZid, newZid),
		ContentPath: renamePath(curEntry.ContentPath, curZid, newZid),
		ContentExt:  curEntry.ContentExt,
	}

	if err = dp.dirSrv.RenameEntry(curEntry, &newEntry); err != nil {
		return err
	}
	oldMeta.Zid = newZid
	newZettel := domain.Zettel{Meta: oldMeta, Content: domain.NewContent(oldContent)}
	if err = setZettel(dp, &newEntry, newZettel); err != nil {
		// "Rollback" rename. No error checking...
		dp.dirSrv.RenameEntry(&newEntry, curEntry)
		return err
	}
	err = deleteZettel(dp, curEntry, curZid)
	if err == nil {
		dp.notifyChanged(change.OnDelete, curZid)
		dp.notifyChanged(change.OnUpdate, newZid)
	}
	return err
}

func (dp *dirPlace) CanDeleteZettel(ctx context.Context, zid id.Zid) bool {
	if dp.readonly {
		return false
	}
	entry, err := dp.dirSrv.GetEntry(zid)
	return err == nil && entry.IsValid()
}

func (dp *dirPlace) DeleteZettel(ctx context.Context, zid id.Zid) error {
	if dp.readonly {
		return place.ErrReadOnly
	}

	entry, err := dp.dirSrv.GetEntry(zid)
	if err != nil || !entry.IsValid() {
		return nil
	}
	dp.dirSrv.DeleteEntry(zid)
	err = deleteZettel(dp, entry, zid)
	if err == nil {
		dp.notifyChanged(change.OnDelete, zid)
	}
	return err
}

func (dp *dirPlace) ReadStats(st *place.Stats) {
	st.ReadOnly = dp.readonly
	st.Zettel, _ = dp.dirSrv.NumEntries()
}

func (dp *dirPlace) cleanupMeta(ctx context.Context, m *meta.Meta) {
	if role, ok := m.Get(meta.KeyRole); !ok || role == "" {
		m.Set(meta.KeyRole, runtime.GetDefaultRole())
	}
	if syntax, ok := m.Get(meta.KeySyntax); !ok || syntax == "" {

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package dirplace provides a directory-based zettel place.
package dirplace

import (
	"zettelstore.de/z/config/startup"
	"zettelstore.de/z/place/dirplace/notifydir"
	"zettelstore.de/z/place/dirplace/simpledir"
)

func getDirSrvInfo(dirType string) (directoryServiceSpec, int, int) {
	for count := 0; count < 2; count++ {
		switch dirType {
		case startup.ValueDirPlaceTypeNotify:
			return dirSrvNotify, 7, 1499
		case startup.ValueDirPlaceTypeSimple:
			return dirSrvSimple, 1, 1
		default:
			dirType = startup.DefaultDirPlaceType()
		}
	}
	panic("unable to set default dir place type: " + dirType)
}

func (dp *dirPlace) setupDirService() {
	switch dp.dirSrvSpec {
	case dirSrvSimple:
		dp.dirSrv = simpledir.NewService(dp.dir)
		dp.mustNotify = true
	default:
		dp.dirSrv = notifydir.NewService(dp.dir, dp.dirRescan, dp.cdata.Notify)
		dp.mustNotify = false
	}
}

//-----------------------------------------------------------------------------
// Copyright (c) 2020-2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package notifydir manages the notified directory part of a dirstore.
package notifydir

import (
	"time"

	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/place/change"
	"zettelstore.de/z/place/dirplace/directory"
)

// notifyService specifies a directory scan service.
type notifyService struct {
	dirPath    string
	rescanTime time.Duration
	done       chan struct{}
	cmds       chan dirCmd
	infos      chan<- change.Info
}

// NewService creates a new directory service.
func NewService(directoryPath string, rescanTime time.Duration, chci chan<- change.Info) directory.Service {
	srv := &notifyService{
		dirPath:    directoryPath,
		rescanTime: rescanTime,
		cmds:       make(chan dirCmd),
		infos:      chci,
	}
	return srv
}

// Start makes the directory service operational.
func (srv *notifyService) Start() error {
	tick := make(chan struct{})
	rawEvents := make(chan *fileEvent)
	events := make(chan *fileEvent)

	ready := make(chan int)
	go srv.directoryService(events, ready)
	go collectEvents(events, rawEvents)
	go watchDirectory(srv.dirPath, rawEvents, tick)

	if srv.done != nil {
		panic("src.done already set")
	}
	srv.done = make(chan struct{})
	go ping(tick, srv.rescanTime, srv.done)
	<-ready
	return nil
}

// Stop stops the directory service.
func (srv *notifyService) Stop() error {
	close(srv.done)
	srv.done = nil
	return nil
}

func (srv *notifyService) notifyChange(reason change.Reason, zid id.Zid) {
	if chci := srv.infos; chci != nil {
		chci <- change.Info{Reason: reason, Zid: zid}
	}
}

// NumEntries returns the number of managed zettel.
func (srv *notifyService) NumEntries() (int, error) {
	resChan := make(chan resNumEntries)
	srv.cmds <- &cmdNumEntries{resChan}
	return <-resChan, nil
}

// GetEntries returns an unsorted list of all current directory entries.
func (srv *notifyService) GetEntries() ([]*directory.Entry, error) {
	resChan := make(chan resGetEntries)
	srv.cmds <- &cmdGetEntries{resChan}
	return <-resChan, nil
}

// GetEntry returns the entry with the specified zettel id. If there is no such
// zettel id, an empty entry is returned.
func (srv *notifyService) GetEntry(zid id.Zid) (*directory.Entry, error) {
	resChan := make(chan resGetEntry)
	srv.cmds <- &cmdGetEntry{zid, resChan}
	return <-resChan, nil
}

// GetNew returns an entry with a new zettel id.
func (srv *notifyService) GetNew() (*directory.Entry, error) {
	resChan := make(chan resNewEntry)
	srv.cmds <- &cmdNewEntry{resChan}
	result := <-resChan
	return result.entry, result.err
}

// UpdateEntry notifies the directory of an updated entry.
func (srv *notifyService) UpdateEntry(entry *directory.Entry) error {
	resChan := make(chan struct{})
	srv.cmds <- &cmdUpdateEntry{entry, resChan}
	<-resChan
	return nil
}

// RenameEntry notifies the directory of an renamed entry.
func (srv *notifyService) RenameEntry(curEntry, newEntry *directory.Entry) error {
	resChan := make(chan resRenameEntry)
	srv.cmds <- &cmdRenameEntry{curEntry, newEntry, resChan}
	return <-resChan
}

// DeleteEntry removes a zettel id from the directory of entries.
func (srv *notifyService) DeleteEntry(zid id.Zid) error {
	resChan := make(chan struct{})
	srv.cmds <- &cmdDeleteEntry{zid, resChan}
	<-resChan
	return nil
}

//-----------------------------------------------------------------------------
// Copyright (c) 2020-2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package notifydir manages the notified directory part of a dirstore.
package notifydir

import (
	"log"
	"time"

	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/place"
	"zettelstore.de/z/place/change"
	"zettelstore.de/z/place/dirplace/directory"
)

// ping sends every tick a signal to reload the directory list
func ping(tick chan<- struct{}, rescanTime time.Duration, done <-chan struct{}) {
	ticker := time.NewTicker(rescanTime)
	defer close(tick)
	for {
		select {
		case _, ok := <-ticker.C:
			if !ok {
				return
			}
			tick <- struct{}{}
		case _, ok := <-done:
			if !ok {
				ticker.Stop()
				return
			}
		}
	}
}

func newEntry(ev *fileEvent) *directory.Entry {
	de := new(directory.Entry)
	de.Zid = ev.zid
	updateEntry(de, ev)
	return de
}

func updateEntry(de *directory.Entry, ev *fileEvent) {
	if ev.ext == "meta" {
		de.MetaSpec = directory.MetaSpecFile
		de.MetaPath = ev.path
		return
	}
	if de.ContentExt != "" && de.ContentExt != ev.ext {
		de.Duplicates = true
		return
	}
	if de.MetaSpec != directory.MetaSpecFile {
		if ev.ext == "zettel" {
			de.MetaSpec = directory.MetaSpecHeader
		} else {
			de.MetaSpec = directory.MetaSpecNone
		}
	}
	de.ContentPath = ev.path
	de.ContentExt = ev.ext
}

type dirMap map[id.Zid]*directory.Entry

func dirMapUpdate(dm dirMap, ev *fileEvent) {
	de := dm[ev.zid]
	if de == nil {
		dm[ev.zid] = newEntry(ev)
		return
	}
	updateEntry(de, ev)
}

func deleteFromMap(dm dirMap, ev *fileEvent) {
	if ev.ext == "meta" {
		if entry, ok := dm[ev.zid]; ok {
			if entry.MetaSpec == directory.MetaSpecFile {
				entry.MetaSpec = directory.MetaSpecNone
				return
			}
		}
	}
	delete(dm, ev.zid)
}

// directoryService is the main service.
func (srv *notifyService) directoryService(events <-chan *fileEvent, ready chan<- int) {
	curMap := make(dirMap)
	var newMap dirMap
	for {
		select {
		case ev, ok := <-events:
			if !ok {
				return
			}
			switch ev.status {
			case fileStatusReloadStart:
				newMap = make(dirMap)
			case fileStatusReloadEnd:
				curMap = newMap
				newMap = nil
				if ready != nil {
					ready <- len(curMap)
					close(ready)
					ready = nil
				}
				srv.notifyChange(change.OnReload, id.Invalid)
			case fileStatusError:
				log.Println("DIRPLACE", "ERROR", ev.err)
			case fileStatusUpdate:
				srv.processFileUpdateEvent(ev, curMap, newMap)
			case fileStatusDelete:
				srv.processFileDeleteEvent(ev, curMap, newMap)
			}
		case cmd, ok := <-srv.cmds:
			if ok {
				cmd.run(curMap)
			}
		}
	}
}

func (srv *notifyService) processFileUpdateEvent(ev *fileEvent, curMap, newMap dirMap) {
	if newMap != nil {
		dirMapUpdate(newMap, ev)
	} else {
		dirMapUpdate(curMap, ev)
		srv.notifyChange(change.OnUpdate, ev.zid)
	}
}

func (srv *notifyService) processFileDeleteEvent(ev *fileEvent, curMap, newMap dirMap) {
	if newMap != nil {
		deleteFromMap(newMap, ev)
	} else {
		deleteFromMap(curMap, ev)
		srv.notifyChange(change.OnDelete, ev.zid)
	}
}

type dirCmd interface {
	run(m dirMap)
}

type cmdNumEntries struct {
	result chan<- resNumEntries
}
type resNumEntries = int

func (cmd *cmdNumEntries) run(m dirMap) {
	cmd.result <- len(m)
}

type cmdGetEntries struct {
	result chan<- resGetEntries
}
type resGetEntries []*directory.Entry

func (cmd *cmdGetEntries) run(m dirMap) {
	res := make([]*directory.Entry, len(m))
	i := 0
	for _, de := range m {
		entry := *de
		res[i] = &entry
		i++
	}
	cmd.result <- res
}

type cmdGetEntry struct {
	zid    id.Zid
	result chan<- resGetEntry
}
type resGetEntry = *directory.Entry

func (cmd *cmdGetEntry) run(m dirMap) {
	entry := m[cmd.zid]
	if entry == nil {
		cmd.result <- nil
	} else {
		result := *entry
		cmd.result <- &result
	}
}

type cmdNewEntry struct {
	result chan<- resNewEntry
}
type resNewEntry struct {
	entry *directory.Entry
	err   error
}

func (cmd *cmdNewEntry) run(m dirMap) {
	zid, err := place.GetNewZid(func(zid id.Zid) (bool, error) {
		_, ok := m[zid]
		return !ok, nil
	})
	if err != nil {
		cmd.result <- resNewEntry{nil, err}
		return
	}
	entry := &directory.Entry{Zid: zid}
	m[zid] = entry
	cmd.result <- resNewEntry{&directory.Entry{Zid: zid}, nil}
}

type cmdUpdateEntry struct {
	entry  *directory.Entry
	result chan<- struct{}
}

func (cmd *cmdUpdateEntry) run(m dirMap) {
	entry := *cmd.entry
	m[entry.Zid] = &entry
	cmd.result <- struct{}{}
}

type cmdRenameEntry struct {
	curEntry *directory.Entry
	newEntry *directory.Entry
	result   chan<- resRenameEntry
}

type resRenameEntry = error

func (cmd *cmdRenameEntry) run(m dirMap) {
	newEntry := *cmd.newEntry
	newZid := newEntry.Zid
	if _, found := m[newZid]; found {
		cmd.result <- &place.ErrInvalidID{Zid: newZid}
		return
	}
	delete(m, cmd.curEntry.Zid)
	m[newZid] = &newEntry
	cmd.result <- nil
}

type cmdDeleteEntry struct {
	zid    id.Zid
	result chan<- struct{}
}

func (cmd *cmdDeleteEntry) run(m dirMap) {
	delete(m, cmd.zid)
	cmd.result <- struct{}{}
}

//-----------------------------------------------------------------------------
// Copyright (c) 2020-2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package notifydir manages the notified directory part of a dirstore.
package notifydir

import (
	"os"
	"path/filepath"
	"regexp"
	"time"

	"github.com/fsnotify/fsnotify"

	"zettelstore.de/z/domain/id"
)

var validFileName = regexp.MustCompile(`^(\d{14}).*(\.(.+))$`)

func matchValidFileName(name string) []string {
	return validFileName.FindStringSubmatch(name)
}

type fileStatus int

const (
	fileStatusNone fileStatus = iota
	fileStatusReloadStart
	fileStatusReloadEnd
	fileStatusError
	fileStatusUpdate
	fileStatusDelete
)

type fileEvent struct {
	status fileStatus
	path   string // Full file path
	zid    id.Zid
	ext    string // File extension
	err    error  // Error if Status == fileStatusError
}

type sendResult int

const (
	sendDone sendResult = iota
	sendReload
	sendExit
)

func watchDirectory(directory string, events chan<- *fileEvent, tick <-chan struct{}) {
	defer close(events)

	var watcher *fsnotify.Watcher
	defer func() {
		if watcher != nil {
			watcher.Close()
		}
	}()

	sendEvent := func(ev *fileEvent) sendResult {
		select {
		case events <- ev:
		case _, ok := <-tick:
			if ok {
				return sendReload
			}
			return sendExit
		}
		return sendDone
	}

	sendError := func(err error) sendResult {
		return sendEvent(&fileEvent{status: fileStatusError, err: err})
	}

	sendFileEvent := func(status fileStatus, path string, match []string) sendResult {
		zid, err := id.Parse(match[1])
		if err != nil {
			return sendDone
		}
		event := &fileEvent{
			status: status,
			path:   path,
			zid:    zid,
			ext:    match[3],
		}
		return sendEvent(event)
	}

	reloadStartEvent := &fileEvent{status: fileStatusReloadStart}
	reloadEndEvent := &fileEvent{status: fileStatusReloadEnd}
	reloadFiles := func() bool {
		entries, err := os.ReadDir(directory)
		if err != nil {
			if res := sendError(err); res != sendDone {
				return res == sendReload
			}
			return true
		}

		if res := sendEvent(reloadStartEvent); res != sendDone {
			return res == sendReload
		}

		if watcher != nil {
			watcher.Close()
		}
		watcher, err = fsnotify.NewWatcher()
		if err != nil {
			if res := sendError(err); res != sendDone {
				return res == sendReload
			}
		}

		for _, entry := range entries {
			if entry.IsDir() {
				continue
			}
			if info, err1 := entry.Info(); err1 != nil || !info.Mode().IsRegular() {
				continue
			}
			name := entry.Name()
			match := matchValidFileName(name)
			if len(match) > 0 {
				path := filepath.Join(directory, name)
				if res := sendFileEvent(fileStatusUpdate, path, match); res != sendDone {
					return res == sendReload
				}
			}
		}

		if watcher != nil {
			err = watcher.Add(directory)
			if err != nil {
				if res := sendError(err); res != sendDone {
					return res == sendReload
				}
			}
		}
		if res := sendEvent(reloadEndEvent); res != sendDone {
			return res == sendReload
		}
		return true
	}

	handleEvents := func() bool {
		const createOps = fsnotify.Create | fsnotify.Write
		const deleteOps = fsnotify.Remove | fsnotify.Rename

		for {
			select {
			case wevent, ok := <-watcher.Events:
				if !ok {
					return false
				}
				path := filepath.Clean(wevent.Name)
				match := matchValidFileName(filepath.Base(path))
				if len(match) == 0 {
					continue
				}
				if wevent.Op&createOps != 0 {
					if fi, err := os.Lstat(path); err != nil || !fi.Mode().IsRegular() {
						continue
					}
					if res := sendFileEvent(
						fileStatusUpdate, path, match); res != sendDone {
						return res == sendReload
					}
				}
				if wevent.Op&deleteOps != 0 {
					if res := sendFileEvent(
						fileStatusDelete, path, match); res != sendDone {
						return res == sendReload
					}
				}
			case err, ok := <-watcher.Errors:
				if !ok {
					return false
				}
				if res := sendError(err); res != sendDone {
					return res == sendReload
				}
			case _, ok := <-tick:
				return ok
			}
		}
	}

	for {
		if !reloadFiles() {
			return
		}
		if watcher == nil {
			if _, ok := <-tick; !ok {
				return
			}
		} else {
			if !handleEvents() {
				return
			}
		}
	}
}

func sendCollectedEvents(out chan<- *fileEvent, events []*fileEvent) {
	for _, ev := range events {
		if ev.status != fileStatusNone {
			out <- ev
		}
	}
}

func addEvent(events []*fileEvent, ev *fileEvent) []*fileEvent {
	switch ev.status {
	case fileStatusNone:
		return events
	case fileStatusReloadStart:
		events = events[0:0]
	case fileStatusUpdate, fileStatusDelete:
		if len(events) > 0 && mergeEvents(events, ev) {
			return events
		}
	}
	return append(events, ev)
}

func mergeEvents(events []*fileEvent, ev *fileEvent) bool {
	for i := len(events) - 1; i >= 0; i-- {
		oev := events[i]
		switch oev.status {
		case fileStatusReloadStart, fileStatusReloadEnd:
			return false
		case fileStatusUpdate, fileStatusDelete:
			if ev.path == oev.path {
				if ev.status == oev.status {
					return true
				}
				oev.status = fileStatusNone
				return false
			}
		}
	}
	return false
}

func collectEvents(out chan<- *fileEvent, in <-chan *fileEvent) {
	defer close(out)

	var sendTime time.Time
	sendTimeSet := false
	ticker := time.NewTicker(500 * time.Millisecond)
	defer ticker.Stop()

	events := make([]*fileEvent, 0, 32)
	buffer := false
	for {
		select {
		case ev, ok := <-in:
			if !ok {
				sendCollectedEvents(out, events)
				return
			}
			if ev.status == fileStatusReloadStart {
				buffer = false
				events = events[0:0]
			}
			if buffer {
				if !sendTimeSet {
					sendTime = time.Now().Add(1500 * time.Millisecond)
					sendTimeSet = true
				}
				events = addEvent(events, ev)
				if len(events) > 1024 {
					sendCollectedEvents(out, events)
					events = events[0:0]
					sendTimeSet = false
				}
				continue
			}
			out <- ev
			if ev.status == fileStatusReloadEnd {
				buffer = true
			}
		case now := <-ticker.C:
			if sendTimeSet && now.After(sendTime) {
				sendCollectedEvents(out, events)
				events = events[0:0]
				sendTimeSet = false
			}
		}
	}
}

//-----------------------------------------------------------------------------
// Copyright (c) 2020 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package notifydir manages the notified directory part of a dirstore.
package notifydir

import "testing"

func sameStringSlices(sl1, sl2 []string) bool {
	if len(sl1) != len(sl2) {
		return false
	}
	for i := 0; i < len(sl1); i++ {
		if sl1[i] != sl2[i] {
			return false
		}
	}
	return true
}

func TestMatchValidFileName(t *testing.T) {
	testcases := []struct {
		name string
		exp  []string
	}{
		{"", []string{}},
		{".txt", []string{}},
		{"12345678901234.txt", []string{"12345678901234", ".txt", "txt"}},
		{"12345678901234abc.txt", []string{"12345678901234", ".txt", "txt"}},
		{"12345678901234.abc.txt", []string{"12345678901234", ".txt", "txt"}},
	}

	for i, tc := range testcases {
		got := matchValidFileName(tc.name)
		if len(got) == 0 {
			if len(tc.exp) > 0 {
				t.Errorf("TC=%d, name=%q, exp=%v, got=%v", i, tc.name, tc.exp, got)
			}
		} else {
			if got[0] != tc.name {
				t.Errorf("TC=%d, name=%q, got=%v", i, tc.name, got)
			}
			if !sameStringSlices(got[1:], tc.exp) {
				t.Errorf("TC=%d, name=%q, exp=%v, got=%v", i, tc.name, tc.exp, got)
			}
		}
	}
}

		err = cmd.runMetaSpecFile()
	case directory.MetaSpecHeader:
		err = cmd.runMetaSpecHeader()
	case directory.MetaSpecNone:
		// TODO: if meta has some additional infos: write meta to new .meta;
		// update entry in dir
		err = writeFileContent(cmd.entry.ContentPath, cmd.zettel.Content.AsString())
	default:
		panic("TODO: ???")
	}
	cmd.rc <- err
}

func (cmd *fileSetZettel) runMetaSpecFile() error {
	f, err := openFileWrite(cmd.entry.MetaPath)

		if err == nil {
			err = err1
		}
	case directory.MetaSpecHeader:
		err = os.Remove(cmd.entry.ContentPath)
	case directory.MetaSpecNone:
		err = os.Remove(cmd.entry.ContentPath)
	default:
		panic("TODO: ???")
	}
	cmd.rc <- err
}

// Utility functions ----------------------------------------

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package simpledir manages the directory part of a dirstore.
package simpledir

import (
	"os"
	"path/filepath"
	"regexp"
	"sync"

	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/place"
	"zettelstore.de/z/place/dirplace/directory"
)

// simpleService specifies a directory service without scanning.
type simpleService struct {
	dirPath string
	mx      sync.Mutex
}

// NewService creates a new directory service.
func NewService(directoryPath string) directory.Service {
	return &simpleService{
		dirPath: directoryPath,
	}
}

func (ss *simpleService) Start() error {
	ss.mx.Lock()
	defer ss.mx.Unlock()
	_, err := os.ReadDir(ss.dirPath)
	return err
}

func (ss *simpleService) Stop() error {
	return nil
}

func (ss *simpleService) NumEntries() (int, error) {
	ss.mx.Lock()
	defer ss.mx.Unlock()
	entries, err := ss.getEntries()
	if err == nil {
		return len(entries), nil
	}
	return 0, err
}

func (ss *simpleService) GetEntries() ([]*directory.Entry, error) {
	ss.mx.Lock()
	defer ss.mx.Unlock()
	entrySet, err := ss.getEntries()
	if err != nil {
		return nil, err
	}
	result := make([]*directory.Entry, 0, len(entrySet))
	for _, entry := range entrySet {
		result = append(result, entry)
	}
	return result, nil
}
func (ss *simpleService) getEntries() (map[id.Zid]*directory.Entry, error) {
	dirEntries, err := os.ReadDir(ss.dirPath)
	if err != nil {
		return nil, err
	}
	entrySet := make(map[id.Zid]*directory.Entry)
	for _, dirEntry := range dirEntries {
		if dirEntry.IsDir() {
			continue
		}
		if info, err1 := dirEntry.Info(); err1 != nil || !info.Mode().IsRegular() {
			continue
		}
		name := dirEntry.Name()
		match := matchValidFileName(name)
		if len(match) == 0 {
			continue
		}
		zid, err := id.Parse(match[1])
		if err != nil {
			continue
		}
		var entry *directory.Entry
		if e, ok := entrySet[zid]; ok {
			entry = e
		} else {
			entry = &directory.Entry{Zid: zid}
			entrySet[zid] = entry
		}
		updateEntry(entry, filepath.Join(ss.dirPath, name), match[3])
	}
	return entrySet, nil
}

var validFileName = regexp.MustCompile(`^(\d{14}).*(\.(.+))$`)

func matchValidFileName(name string) []string {
	return validFileName.FindStringSubmatch(name)
}

func updateEntry(entry *directory.Entry, path, ext string) {
	if ext == "meta" {
		entry.MetaSpec = directory.MetaSpecFile
		entry.MetaPath = path
	} else if entry.ContentExt != "" && entry.ContentExt != ext {
		entry.Duplicates = true
	} else {
		if entry.MetaSpec != directory.MetaSpecFile {
			if ext == "zettel" {
				entry.MetaSpec = directory.MetaSpecHeader
			} else {
				entry.MetaSpec = directory.MetaSpecNone
			}
		}
		entry.ContentPath = path
		entry.ContentExt = ext
	}
}

func (ss *simpleService) GetEntry(zid id.Zid) (*directory.Entry, error) {
	ss.mx.Lock()
	defer ss.mx.Unlock()
	return ss.getEntry(zid)
}
func (ss *simpleService) getEntry(zid id.Zid) (*directory.Entry, error) {
	pattern := filepath.Join(ss.dirPath, zid.String()) + "*.*"
	paths, err := filepath.Glob(pattern)
	if err != nil {
		return nil, err
	}
	if len(paths) == 0 {
		return nil, nil
	}
	entry := &directory.Entry{Zid: zid}
	for _, path := range paths {
		ext := filepath.Ext(path)
		if len(ext) > 0 && ext[0] == '.' {
			ext = ext[1:]
		}
		updateEntry(entry, path, ext)
	}
	return entry, nil
}

func (ss *simpleService) GetNew() (*directory.Entry, error) {
	ss.mx.Lock()
	defer ss.mx.Unlock()
	zid, err := place.GetNewZid(func(zid id.Zid) (bool, error) {
		entry, err := ss.getEntry(zid)
		if err != nil {
			return false, nil
		}
		return !entry.IsValid(), nil
	})
	if err != nil {
		return nil, err
	}
	return &directory.Entry{Zid: zid}, nil
}

func (ss *simpleService) UpdateEntry(entry *directory.Entry) error {
	// Noting to to, since the actual file update is done by dirplace
	return nil
}

func (ss *simpleService) RenameEntry(curEntry, newEntry *directory.Entry) error {
	// Noting to to, since the actual file rename is done by dirplace
	return nil
}

func (ss *simpleService) DeleteEntry(zid id.Zid) error {
	// Noting to to, since the actual file delete is done by dirplace
	return nil
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package place provides a generic interface to zettel places.
package place

import (
	"time"

	"zettelstore.de/z/domain/id"
)

// GetNewZid calculates a new and unused zettel identifier, based on the current date and time.
func GetNewZid(testZid func(id.Zid) (bool, error)) (id.Zid, error) {
	withSeconds := false
	for i := 0; i < 90; i++ { // Must be completed within 9 seconds (less than web/server.writeTimeout)
		zid := id.New(withSeconds)
		found, err := testZid(zid)
		if err != nil {
			return id.Invalid, err
		}
		if found {
			return zid, nil
		}
		// TODO: do not wait here unconditionally.
		time.Sleep(100 * time.Millisecond)
		withSeconds = true
	}
	return id.Invalid, ErrTimeout
}

// Package memplace stores zettel volatile in main memory.
package memplace

import (
	"context"
	"net/url"
	"sync"


	"zettelstore.de/z/domain"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/place"
	"zettelstore.de/z/place/change"
	"zettelstore.de/z/place/manager"

	return nil
}

func (mp *memPlace) CanCreateZettel(ctx context.Context) bool { return true }

func (mp *memPlace) CreateZettel(ctx context.Context, zettel domain.Zettel) (id.Zid, error) {
	mp.mx.Lock()
	zid, err := place.GetNewZid(func(zid id.Zid) (bool, error) {
		_, ok := mp.zettel[zid]
		return !ok, nil
	})
	if err != nil {
		mp.mx.Unlock()
		return id.Invalid, err
	}
	meta := zettel.Meta.Clone()
	meta.Zid = zid
	zettel.Meta = meta
	mp.zettel[zid] = zettel
	mp.mx.Unlock()
	mp.notifyChanged(change.OnUpdate, zid)






	return zid, nil








}

func (mp *memPlace) GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error) {
	mp.mx.RLock()
	zettel, ok := mp.zettel[zid]
	mp.mx.RUnlock()
	if !ok {

var ErrStopped = errors.New("place is stopped")

// ErrReadOnly is returned if there is an attepmt to write to a read-only place.
var ErrReadOnly = errors.New("read-only place")

// ErrNotFound is returned if a zettel was not found in the place.
var ErrNotFound = errors.New("zettel not found")

// ErrTimeout is returned if a place operation takes too long.
// One example: if calculating a new zettel identifier takes too long.
var ErrTimeout = errors.New("timeout")

// ErrInvalidID is returned if the zettel id is not appropriate for the place operation.
type ErrInvalidID struct{ Zid id.Zid }

func (err *ErrInvalidID) Error() string { return "invalid Zettel id: " + err.Zid.String() }

	fmt.Fprintf(&sb, "|Listen address| %v\n", startup.ListenAddress())
	fmt.Fprintf(&sb, "|Authentication enabled|%v\n", startup.WithAuth())
	fmt.Fprintf(&sb, "|Secure cookie|%v\n", startup.SecureCookie())
	fmt.Fprintf(&sb, "|Persistent Cookie|%v\n", startup.PersistentCookie())
	html, api := startup.TokenLifetime()
	fmt.Fprintf(&sb, "|API Token lifetime|%v\n", api)
	fmt.Fprintf(&sb, "|HTML Token lifetime|%v\n", html)
	fmt.Fprintf(&sb, "|Default directory place type|%v", startup.DefaultDirPlaceType())
	return sb.String()
}

// Package tests provides some higher-level tests.
package tests

import (
	"context"
	"fmt"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"zettelstore.de/z/ast"
	"zettelstore.de/z/config/startup"
	"zettelstore.de/z/domain"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/encoder"
	"zettelstore.de/z/parser"
	"zettelstore.de/z/place"
	"zettelstore.de/z/place/manager"

		panic(err)
	}

	cdata := manager.ConnectData{Filter: &noFilter{}, Notify: nil}
	for _, entry := range entries {
		if entry.IsDir() {
			place, err := manager.Connect(
				"dir://"+filepath.Join(root, entry.Name())+"?type="+startup.ValueDirPlaceTypeSimple,
				false,
				&cdata,
			)
			if err != nil {
				panic(err)
			}
			places = append(places, place)

	if gotFirst != gotSecond {
		t.Errorf("\n1st: %q\n2nd: %q", gotFirst, gotSecond)
	}
}

func getPlaceName(p place.ManagedPlace, root string) string {
	u, err := url.Parse(p.Location())
	if err != nil {
		panic("Unable to parse URL '" + p.Location() + "': " + err.Error())
	}
	return u.Path[len(root):]
}

func match(*meta.Meta) bool { return true }

func checkContentPlace(t *testing.T, p place.ManagedPlace, wd, placeName string) {
	ss := p.(place.StartStopper)
	if err := ss.Start(context.Background()); err != nil {

	}
	if err1, ok := err.(*ErrBadRequest); ok {
		return http.StatusBadRequest, err1.Text
	}
	if err == place.ErrStopped {
		return http.StatusInternalServerError, fmt.Sprintf("Zettelstore not operational: %v.", err)
	}
	if err == place.ErrTimeout {
		return http.StatusLoopDetected, "Zettelstore operation took too long"
	}
	return http.StatusInternalServerError, err.Error()
}

<title>Change Log</title>

<a name="0_0_13"></a>
<h2>Changes for Version 0.0.13 (pending)</h2>

<a name="0_0_12"></a>
<h2>Changes for Version 0.0.12 (2021-04-16)</h2>
  *  Raise the per-process limit of open files on macOS to 1.048.576. This
     allows most macOS users to use at least 500.000 zettel. That should be
     enough for the near future.
     (major)
  *  Mitigate the shortcomings of the macOS version by introducing types of
     directory places. The original directory place type is now called "notify"
     (the default value). There is a new type called "simple". This new type
     does not notify Zettelstore when some of the underlying Zettel files
     change.
     (major)
  *  Add new startup configuration <tt>default-dir-place-type</tt>, which gives
     the default value for specifying a directory place type. The default value
     is &ldquo;notify&rdquo;. On macOS, the default value may be changed
     &ldquo;simple&rdquo; if some errors occur while raising the per-process
     limit of open files.
     (minor)

<a name="0_0_11"></a>
<h2>Changes for Version 0.0.11 (2021-04-05)</h2>
  *  New place schema "file" allows to read zettel from a ZIP file.
     A zettel collection can now be packaged and distributed easier.
     (major: server)
  *  Non-restricted search is a full-text search. The search string will

     or the whole string. If a search value begins with '#', only zettel with the exact
     tag will be returned. Otherwise a zettel will be returned if the search string
     just matches the prefix of only one of its tags.
     (minor: api, webui)
  *  Many smaller bug fixes and inprovements, to the software and to the documentation.

A note for users of macOS: in the current release and with macOS's default values,
a zettel directory place must not contain more than approx. 250 files. There are three options
to mitigate this limitation temporarily:


  #  You update the per-process limit of open files on macOS.
  #  You setup a virtualization environment to run Zettelstore on Linux or Windows.
  #  You wait for version 0.0.12 which addresses this issue.

<a name="0_0_10"></a>
<h2>Changes for Version 0.0.10 (2021-02-26)</h2>
  *  Menu item &ldquo;Home&rdquo; now redirects to a home zettel.

<title>Download</title>
<h1>Download of Zettelstore Software</h1>
<h2>Foreword</h2>
  *  Zettelstore is free/libre open source software, licensed under EUPL-1.2-or-later.
  *  The software is provided as-is.
  *  There is no guarantee that it will not damage your system.
  *  However, it is in use by the main developer since March 2020 without any damage.
  *  It may be useful for you. It is useful for me.
  *  Take a look at the [https://zettelstore.de/manual/|manual] to know how to start and use it.

<h2>ZIP-ped Executables</h2>
Build: <code>v0.0.12</code> (2021-04-16).

  *  [/uv/zettelstore-0.0.12-linux-amd64.zip|Linux] (amd64)
  *  [/uv/zettelstore-0.0.12-linux-arm.zip|Linux] (arm6, e.g. Raspberry Pi)
  *  [/uv/zettelstore-0.0.12-windows-amd64.zip|Windows] (amd64)
  *  [/uv/zettelstore-0.0.12-darwin-amd64.zip|macOS] (amd64)
  *  [/uv/zettelstore-0.0.12-darwin-arm64.zip|macOS] (arm64, aka Apple silicon)

Unzip the appropriate file, install and execute Zettelstore according to the manual.

<h2>Zettel for the manual</h2>
As a starter, you can download the zettel for the manual [/uv/manual-0.0.11.zip|here].
Just unzip the contained files and put them into your zettel folder or configure
a file place to read the zettel directly from the ZIP file.

It is a live example of the zettelstore software, running in read-only mode.

The software, including the manual, is licensed under the
[/file?name=LICENSE.txt&ci=trunk|European Union Public License 1.2 (or later)].

[https://twitter.com/zettelstore|Stay tuned]…
<hr>
<h3>Latest Release: 0.0.12 (2021-04-16)</h3>
  *  [./download.wiki|Download]
  *  [./changes.wiki#0_0_12|Change Summary]
  *  [/timeline?p=version-0.0.12&bt=version-0.0.11&y=ci|Check-ins for version 0.0.12],
     [/vdiff?to=version-0.0.12&from=version-0.0.11|content diff]
  *  [/timeline?df=version-0.0.12&y=ci|Check-ins derived from the 0.0.12 release],
     [/vdiff?from=version-0.0.12&to=trunk|content diff]
  *  [./plan.wiki|Limitations and planned Improvements]
  *  [/timeline?t=release|Timeline of all past releases]

<hr>
<h2>Build instructions</h2>
Just install [https://fossil-scm.org|Fossil],
[https://golang.org/dl/|Go] and some Go-based tools. Please read the
[./build.md|instructions] for details.

  *  [/dir?ci=trunk|Source Code]
  *  [/download|Download the source code] as a Tarball or a ZIP file
     (you must [/login|login] as user &quot;anonymous&quot;).