0.0.12

}

// RefState indicates the state of the reference.
type RefState int

// Constants for RefState
const (
	RefStateInvalid  RefState = iota // Invalid Referende
	RefStateZettel                   // Reference to an internal zettel
	RefStateSelf                     // Reference to same zettel with a fragment
	RefStateFound                    // Reference to an existing internal zettel
	RefStateBroken                   // Reference to a non-existing internal zettel
	RefStateHosted                   // Reference to local hosted non-Zettel, without URL change
	RefStateBased                    // Reference to local non-Zettel, to be prefixed
	RefStateExternal                 // Reference to external material

	"net/url"

	"zettelstore.de/z/domain/id"
)

// ParseReference parses a string and returns a reference.
func ParseReference(s string) *Reference {
	if s == "" {

		return &Reference{URL: nil, Value: s, State: RefStateInvalid}
	}
	if state, ok := localState(s); ok {
		if state == RefStateBased {
			s = s[1:]
		}
		u, err := url.Parse(s)

	testcases := []struct {
		link       string
		isZettel   bool
		isExternal bool
		isLocal    bool
	}{
		{"", false, false, false},

		{"http://zettelstore.de/z/ast", false, true, false},
		{"12345678901234", true, false, false},
		{"12345678901234#local", true, false, false},
		{"http://12345678901234", false, true, false},
		{"http://zettelstore.de/z/12345678901234", false, true, false},
		{"http://zettelstore.de/12345678901234", false, true, false},
		{"/12345678901234", false, false, true},

// under this license.
//-----------------------------------------------------------------------------

// Package policy provides some interfaces and implementation for authorization policies.
package policy

import (


	"zettelstore.de/z/domain/meta"
)

type anonPolicy struct {
	simpleMode    bool
	expertMode    func() bool
	getVisibility func(*meta.Meta) meta.Visibility
	pre           Policy
}

func (ap *anonPolicy) CanCreate(user, newMeta *meta.Meta) bool {
	return ap.pre.CanCreate(user, newMeta)
}

func (ap *anonPolicy) CanRead(user, m *meta.Meta) bool {

}

func (ap *anonPolicy) CanDelete(user, m *meta.Meta) bool {
	return ap.pre.CanDelete(user, m) && ap.checkVisibility(m)
}

func (ap *anonPolicy) checkVisibility(m *meta.Meta) bool {
	switch ap.getVisibility(m) {
	case meta.VisibilitySimple:
		return ap.simpleMode || ap.expertMode()
	case meta.VisibilityExpert:
		return ap.expertMode()
	}
	return true
}

// under this license.
//-----------------------------------------------------------------------------

// Package policy provides some interfaces and implementation for authorizsation policies.
package policy

import (
	"zettelstore.de/z/config/runtime"
	"zettelstore.de/z/domain/meta"
)

type defaultPolicy struct{}



func (d *defaultPolicy) CanCreate(user, newMeta *meta.Meta) bool { return true }
func (d *defaultPolicy) CanRead(user, m *meta.Meta) bool         { return true }
func (d *defaultPolicy) CanWrite(user, oldMeta, newMeta *meta.Meta) bool {
	return d.canChange(user, oldMeta)
}
func (d *defaultPolicy) CanRename(user, m *meta.Meta) bool { return d.canChange(user, m) }

		// See owner.go:CanWrite.

		// No authentication: check for owner-like restriction, because the user
		// acts as an owner
		return metaRo != meta.ValueUserRoleOwner && !meta.BoolValue(metaRo)
	}

	userRole := runtime.GetUserRole(user)
	switch metaRo {
	case meta.ValueUserRoleReader:
		return userRole > meta.UserRoleReader
	case meta.ValueUserRoleWriter:
		return userRole > meta.UserRoleWriter
	case meta.ValueUserRoleOwner:
		return userRole > meta.UserRoleOwner
	}
	return !meta.BoolValue(metaRo)
}

// under this license.
//-----------------------------------------------------------------------------

// Package policy provides some interfaces and implementation for authorizsation policies.
package policy

import (
	"zettelstore.de/z/config/runtime"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
)

type ownerPolicy struct {
	expertMode    func() bool
	isOwner       func(id.Zid) bool
	getVisibility func(*meta.Meta) meta.Visibility

	pre           Policy
}

func (o *ownerPolicy) CanCreate(user, newMeta *meta.Meta) bool {
	if user == nil || !o.pre.CanCreate(user, newMeta) {
		return false
	}
	return o.userIsOwner(user) || o.userCanCreate(user, newMeta)
}

func (o *ownerPolicy) userCanCreate(user, newMeta *meta.Meta) bool {
	if runtime.GetUserRole(user) == meta.UserRoleReader {
		return false
	}
	if role, ok := newMeta.Get(meta.KeyRole); ok && role == meta.ValueRoleUser {
		return false
	}
	return true
}

func (o *ownerPolicy) CanRead(user, m *meta.Meta) bool {
	// No need to call o.pre.CanRead(user, meta), because it will always return true.
	// Both the default and the readonly policy allow to read a zettel.
	vis := o.getVisibility(m)
	if res, ok := o.checkVisibility(user, vis); ok {
		return res
	}
	return o.userIsOwner(user) || o.userCanRead(user, m, vis)
}

func (o *ownerPolicy) userCanRead(user, m *meta.Meta, vis meta.Visibility) bool {
	switch vis {
	case meta.VisibilityOwner, meta.VisibilitySimple, meta.VisibilityExpert:
		return false
	case meta.VisibilityPublic:
		return true
	}
	if user == nil {
		return false
	}

	meta.KeyUserRole,
}

func (o *ownerPolicy) CanWrite(user, oldMeta, newMeta *meta.Meta) bool {
	if user == nil || !o.pre.CanWrite(user, oldMeta, newMeta) {
		return false
	}
	vis := o.getVisibility(oldMeta)
	if res, ok := o.checkVisibility(user, vis); ok {
		return res
	}
	if o.userIsOwner(user) {
		return true
	}
	if !o.userCanRead(user, oldMeta, vis) {
		return false
	}
	if role, ok := oldMeta.Get(meta.KeyRole); ok && role == meta.ValueRoleUser {
		// Here we know, that user.Zid == newMeta.Zid (because of userCanRead) and
		// user.Zid == newMeta.Zid (because oldMeta.Zid == newMeta.Zid)
		for _, key := range noChangeUser {
			if oldMeta.GetDefault(key, "") != newMeta.GetDefault(key, "") {
				return false
			}
		}
		return true
	}
	if runtime.GetUserRole(user) == meta.UserRoleReader {
		return false
	}
	return o.userCanCreate(user, newMeta)
}

func (o *ownerPolicy) CanRename(user, m *meta.Meta) bool {
	if user == nil || !o.pre.CanRename(user, m) {
		return false
	}
	if res, ok := o.checkVisibility(user, o.getVisibility(m)); ok {
		return res
	}
	return o.userIsOwner(user)
}

func (o *ownerPolicy) CanDelete(user, m *meta.Meta) bool {
	if user == nil || !o.pre.CanDelete(user, m) {
		return false
	}
	if res, ok := o.checkVisibility(user, o.getVisibility(m)); ok {
		return res
	}
	return o.userIsOwner(user)
}

func (o *ownerPolicy) checkVisibility(user *meta.Meta, vis meta.Visibility) (bool, bool) {
	switch vis {
	case meta.VisibilitySimple, meta.VisibilityExpert:
		return o.userIsOwner(user) && o.expertMode(), true
	}
	return false, false
}

func (o *ownerPolicy) userIsOwner(user *meta.Meta) bool {
	if user == nil {
		return false
	}
	if o.isOwner(user.Zid) {
		return true
	}
	if val, ok := user.Get(meta.KeyUserRole); ok && val == meta.ValueUserRoleOwner {
		return true
	}
	return false
}

//-----------------------------------------------------------------------------

// Package policy provides some interfaces and implementation for authorizsation policies.
package policy

import (
	"context"




	"zettelstore.de/z/domain"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/place"
	"zettelstore.de/z/search"
	"zettelstore.de/z/web/session"
)

// PlaceWithPolicy wraps the given place inside a policy place.
func PlaceWithPolicy(


	place place.Place,
	simpleMode bool,
	withAuth func() bool,
	isReadOnlyMode bool,
	expertMode func() bool,
	isOwner func(id.Zid) bool,
	getVisibility func(*meta.Meta) meta.Visibility,
) (place.Place, Policy) {
	pol := newPolicy(simpleMode, withAuth, isReadOnlyMode, expertMode, isOwner, getVisibility)
	return newPlace(place, pol), pol
}

// polPlace implements a policy place.
type polPlace struct {

	place  place.Place
	policy Policy
}

// newPlace creates a new policy place.
func newPlace(place place.Place, policy Policy) place.Place {
	return &polPlace{

		place:  place,
		policy: policy,
	}
}

func (pp *polPlace) Location() string {
	return pp.place.Location()
}

func (pp *polPlace) CanCreateZettel(ctx context.Context) bool {
	return pp.place.CanCreateZettel(ctx)
}

func (pp *polPlace) CreateZettel(ctx context.Context, zettel domain.Zettel) (id.Zid, error) {
	user := session.GetUser(ctx)
	if pp.policy.CanCreate(user, zettel.Meta) {
		return pp.place.CreateZettel(ctx, zettel)
	}
	return id.Invalid, place.NewErrNotAllowed("Create", user, id.Invalid)
}

func (pp *polPlace) GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error) {
	zettel, err := pp.place.GetZettel(ctx, zid)
	if err != nil {
		return domain.Zettel{}, err
	}
	user := session.GetUser(ctx)
	if pp.policy.CanRead(user, zettel.Meta) {
		return zettel, nil
	}
	return domain.Zettel{}, place.NewErrNotAllowed("GetZettel", user, zid)
}

func (pp *polPlace) GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) {
	m, err := pp.place.GetMeta(ctx, zid)
	if err != nil {
		return nil, err
	}
	user := session.GetUser(ctx)
	if pp.policy.CanRead(user, m) {
		return m, nil
	}
	return nil, place.NewErrNotAllowed("GetMeta", user, zid)
}

func (pp *polPlace) FetchZids(ctx context.Context) (id.Set, error) {
	return nil, place.NewErrNotAllowed("fetch-zids", session.GetUser(ctx), id.Invalid)
}

func (pp *polPlace) SelectMeta(ctx context.Context, s *search.Search) ([]*meta.Meta, error) {
	user := session.GetUser(ctx)
	canRead := pp.policy.CanRead
	s = s.AddPreMatch(func(m *meta.Meta) bool { return canRead(user, m) })
	return pp.place.SelectMeta(ctx, s)
}

func (pp *polPlace) CanUpdateZettel(ctx context.Context, zettel domain.Zettel) bool {
	return pp.place.CanUpdateZettel(ctx, zettel)
}

func (pp *polPlace) UpdateZettel(ctx context.Context, zettel domain.Zettel) error {
	zid := zettel.Meta.Zid
	user := session.GetUser(ctx)
	if !zid.IsValid() {
		return &place.ErrInvalidID{Zid: zid}
	}
	// Write existing zettel
	oldMeta, err := pp.place.GetMeta(ctx, zid)
	if err != nil {
		return err

}

func (pp *polPlace) RenameZettel(ctx context.Context, curZid, newZid id.Zid) error {
	meta, err := pp.place.GetMeta(ctx, curZid)
	if err != nil {
		return err
	}
	user := session.GetUser(ctx)
	if pp.policy.CanRename(user, meta) {
		return pp.place.RenameZettel(ctx, curZid, newZid)
	}
	return place.NewErrNotAllowed("Rename", user, curZid)
}

func (pp *polPlace) CanDeleteZettel(ctx context.Context, zid id.Zid) bool {
	return pp.place.CanDeleteZettel(ctx, zid)
}

func (pp *polPlace) DeleteZettel(ctx context.Context, zid id.Zid) error {
	meta, err := pp.place.GetMeta(ctx, zid)
	if err != nil {
		return err
	}
	user := session.GetUser(ctx)
	if pp.policy.CanDelete(user, meta) {
		return pp.place.DeleteZettel(ctx, zid)
	}
	return place.NewErrNotAllowed("Delete", user, zid)
}

func (pp *polPlace) ReadStats(st *place.Stats) {
	pp.place.ReadStats(st)
}

// under this license.
//-----------------------------------------------------------------------------

// Package policy provides some interfaces and implementation for authorizsation policies.
package policy

import (
	"zettelstore.de/z/domain/id"

	"zettelstore.de/z/domain/meta"
)

// Policy is an interface for checking access authorization.
type Policy interface {
	// User is allowed to create a new zettel.
	CanCreate(user, newMeta *meta.Meta) bool

	// User is allowed to read zettel
	CanRead(user, m *meta.Meta) bool

	// User is allowed to write zettel.
	CanWrite(user, oldMeta, newMeta *meta.Meta) bool

	// User is allowed to rename zettel
	CanRename(user, m *meta.Meta) bool

	// User is allowed to delete zettel
	CanDelete(user, m *meta.Meta) bool
}

// newPolicy creates a policy based on given constraints.
func newPolicy(
	simpleMode bool,
	withAuth func() bool,
	isReadOnlyMode bool,
	expertMode func() bool,
	isOwner func(id.Zid) bool,
	getVisibility func(*meta.Meta) meta.Visibility,
) Policy {
	var pol Policy
	if isReadOnlyMode {
		pol = &roPolicy{}
	} else {
		pol = &defaultPolicy{}
	}
	if withAuth() {
		pol = &ownerPolicy{
			expertMode:    expertMode,
			isOwner:       isOwner,
			getVisibility: getVisibility,

			pre:           pol,
		}
	} else {
		pol = &anonPolicy{
			simpleMode:    simpleMode,
			expertMode:    expertMode,
			getVisibility: getVisibility,
			pre:           pol,
		}
	}
	return &prePolicy{pol}
}

type prePolicy struct {
	post Policy
}

func (p *prePolicy) CanCreate(user, newMeta *meta.Meta) bool {
	return newMeta != nil && p.post.CanCreate(user, newMeta)
}

func (p *prePolicy) CanRead(user, m *meta.Meta) bool {

// Package policy provides some interfaces and implementation for authorizsation policies.
package policy

import (
	"fmt"
	"testing"


	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
)

func TestPolicies(t *testing.T) {
	testScene := []struct {
		simple   bool
		readonly bool
		withAuth bool
		expert   bool
	}{
		{true, true, true, true},
		{true, true, true, false},
		{true, true, false, true},
		{true, true, false, false},
		{true, false, true, true},
		{true, false, true, false},
		{true, false, false, true},
		{true, false, false, false},
		{false, true, true, true},
		{false, true, true, false},
		{false, true, false, true},
		{false, true, false, false},
		{false, false, true, true},
		{false, false, true, false},
		{false, false, false, true},
		{false, false, false, false},
	}
	for _, ts := range testScene {
		var authFunc func() bool

		if ts.withAuth {
			authFunc = withAuth
		} else {
			authFunc = withoutAuth
		}
		var expertFunc func() bool
		if ts.expert {
			expertFunc = expertMode
		} else {
			expertFunc = noExpertMode
		}
		pol := newPolicy(ts.simple, authFunc, ts.readonly, expertFunc, isOwner, getVisibility)
		name := fmt.Sprintf("simple=%v/readonly=%v/withauth=%v/expert=%v",
			ts.simple, ts.readonly, ts.withAuth, ts.expert)
		t.Run(name, func(tt *testing.T) {
			testCreate(tt, pol, ts.simple, ts.withAuth, ts.readonly, ts.expert)
			testRead(tt, pol, ts.simple, ts.withAuth, ts.readonly, ts.expert)
			testWrite(tt, pol, ts.simple, ts.withAuth, ts.readonly, ts.expert)
			testRename(tt, pol, ts.simple, ts.withAuth, ts.readonly, ts.expert)
			testDelete(tt, pol, ts.simple, ts.withAuth, ts.readonly, ts.expert)
		})
	}
}



func withAuth() bool          { return true }

func withoutAuth() bool       { return false }
func expertMode() bool        { return true }
func noExpertMode() bool      { return false }
func isOwner(zid id.Zid) bool { return zid == ownerZid }

























func getVisibility(m *meta.Meta) meta.Visibility {
	if vis, ok := m.Get(meta.KeyVisibility); ok {
		switch vis {
		case meta.ValueVisibilityPublic:
			return meta.VisibilityPublic
		case meta.ValueVisibilityOwner:
			return meta.VisibilityOwner
		case meta.ValueVisibilityExpert:
			return meta.VisibilityExpert
		case meta.ValueVisibilitySimple:
			return meta.VisibilitySimple
		}
	}
	return meta.VisibilityLogin
}

func testCreate(t *testing.T, pol Policy, simple, withAuth, readonly, isExpert bool) {
	t.Helper()
	anonUser := newAnon()
	reader := newReader()
	writer := newWriter()
	owner := newOwner()
	owner2 := newOwner2()
	zettel := newZettel()

			if tc.exp != got {
				tt.Errorf("exp=%v, but got=%v", tc.exp, got)
			}
		})
	}
}

func testRead(t *testing.T, pol Policy, simple, withAuth, readonly, expert bool) {
	t.Helper()
	anonUser := newAnon()
	reader := newReader()
	writer := newWriter()
	owner := newOwner()
	owner2 := newOwner2()
	zettel := newZettel()
	publicZettel := newPublicZettel()
	loginZettel := newLoginZettel()
	ownerZettel := newOwnerZettel()
	expertZettel := newExpertZettel()
	simpleZettel := newSimpleZettel()
	userZettel := newUserZettel()
	testCases := []struct {
		user *meta.Meta
		meta *meta.Meta
		exp  bool
	}{
		// No meta

		{owner2, ownerZettel, true},
		// Expert zettel
		{anonUser, expertZettel, !withAuth && expert},
		{reader, expertZettel, !withAuth && expert},
		{writer, expertZettel, !withAuth && expert},
		{owner, expertZettel, expert},
		{owner2, expertZettel, expert},
		// Simple expert zettel
		{anonUser, simpleZettel, !withAuth && (simple || expert)},
		{reader, simpleZettel, !withAuth && (simple || expert)},
		{writer, simpleZettel, !withAuth && (simple || expert)},
		{owner, simpleZettel, (!withAuth && simple) || expert},
		{owner2, simpleZettel, (!withAuth && simple) || expert},
		// Other user zettel
		{anonUser, userZettel, !withAuth},
		{reader, userZettel, !withAuth},
		{writer, userZettel, !withAuth},
		{owner, userZettel, true},
		{owner2, userZettel, true},
		// Own user zettel

			if tc.exp != got {
				tt.Errorf("exp=%v, but got=%v", tc.exp, got)
			}
		})
	}
}

func testWrite(t *testing.T, pol Policy, simple, withAuth, readonly, expert bool) {
	t.Helper()
	anonUser := newAnon()
	reader := newReader()
	writer := newWriter()
	owner := newOwner()
	owner2 := newOwner2()
	zettel := newZettel()
	publicZettel := newPublicZettel()
	loginZettel := newLoginZettel()
	ownerZettel := newOwnerZettel()
	expertZettel := newExpertZettel()
	simpleZettel := newSimpleZettel()
	userZettel := newUserZettel()
	writerNew := writer.Clone()
	writerNew.Set(meta.KeyUserRole, owner.GetDefault(meta.KeyUserRole, ""))
	roFalse := newRoFalseZettel()
	roTrue := newRoTrueZettel()
	roReader := newRoReaderZettel()
	roWriter := newRoWriterZettel()
	roOwner := newRoOwnerZettel()

	testCases := []struct {
		user *meta.Meta
		old  *meta.Meta
		new  *meta.Meta
		exp  bool
	}{
		// No old and new meta

		// Old an new zettel have different zettel identifier
		{anonUser, zettel, publicZettel, false},
		{reader, zettel, publicZettel, false},
		{writer, zettel, publicZettel, false},
		{owner, zettel, publicZettel, false},
		{owner2, zettel, publicZettel, false},
		// Overwrite a normal zettel
		{anonUser, zettel, zettel, !withAuth && !readonly},
		{reader, zettel, zettel, !withAuth && !readonly},
		{writer, zettel, zettel, !readonly},
		{owner, zettel, zettel, !readonly},
		{owner2, zettel, zettel, !readonly},
		// Public zettel
		{anonUser, publicZettel, publicZettel, !withAuth && !readonly},
		{reader, publicZettel, publicZettel, !withAuth && !readonly},
		{writer, publicZettel, publicZettel, !readonly},
		{owner, publicZettel, publicZettel, !readonly},
		{owner2, publicZettel, publicZettel, !readonly},
		// Login zettel
		{anonUser, loginZettel, loginZettel, !withAuth && !readonly},
		{reader, loginZettel, loginZettel, !withAuth && !readonly},
		{writer, loginZettel, loginZettel, !readonly},
		{owner, loginZettel, loginZettel, !readonly},
		{owner2, loginZettel, loginZettel, !readonly},
		// Owner zettel
		{anonUser, ownerZettel, ownerZettel, !withAuth && !readonly},
		{reader, ownerZettel, ownerZettel, !withAuth && !readonly},
		{writer, ownerZettel, ownerZettel, !withAuth && !readonly},
		{owner, ownerZettel, ownerZettel, !readonly},
		{owner2, ownerZettel, ownerZettel, !readonly},
		// Expert zettel
		{anonUser, expertZettel, expertZettel, !withAuth && !readonly && expert},
		{reader, expertZettel, expertZettel, !withAuth && !readonly && expert},
		{writer, expertZettel, expertZettel, !withAuth && !readonly && expert},
		{owner, expertZettel, expertZettel, !readonly && expert},
		{owner2, expertZettel, expertZettel, !readonly && expert},
		// Simple expert zettel
		{anonUser, simpleZettel, expertZettel, !withAuth && !readonly && (simple || expert)},
		{reader, simpleZettel, expertZettel, !withAuth && !readonly && (simple || expert)},
		{writer, simpleZettel, expertZettel, !withAuth && !readonly && (simple || expert)},
		{owner, simpleZettel, expertZettel, !readonly && ((!withAuth && simple) || expert)},
		{owner2, simpleZettel, expertZettel, !readonly && ((!withAuth && simple) || expert)},
		// Other user zettel
		{anonUser, userZettel, userZettel, !withAuth && !readonly},
		{reader, userZettel, userZettel, !withAuth && !readonly},
		{writer, userZettel, userZettel, !withAuth && !readonly},
		{owner, userZettel, userZettel, !readonly},
		{owner2, userZettel, userZettel, !readonly},
		// Own user zettel
		{reader, reader, reader, !readonly},
		{writer, writer, writer, !readonly},
		{owner, owner, owner, !readonly},
		{owner2, owner2, owner2, !readonly},
		// Writer cannot change importand metadata of its own user zettel
		{writer, writer, writerNew, !withAuth && !readonly},
		// No r/o zettel
		{anonUser, roFalse, roFalse, !withAuth && !readonly},
		{reader, roFalse, roFalse, !withAuth && !readonly},
		{writer, roFalse, roFalse, !readonly},
		{owner, roFalse, roFalse, !readonly},
		{owner2, roFalse, roFalse, !readonly},
		// Reader r/o zettel
		{anonUser, roReader, roReader, false},
		{reader, roReader, roReader, false},
		{writer, roReader, roReader, !readonly},

			if tc.exp != got {
				tt.Errorf("exp=%v, but got=%v", tc.exp, got)
			}
		})
	}
}

func testRename(t *testing.T, pol Policy, simple, withAuth, readonly, expert bool) {
	t.Helper()
	anonUser := newAnon()
	reader := newReader()
	writer := newWriter()
	owner := newOwner()
	owner2 := newOwner2()
	zettel := newZettel()
	expertZettel := newExpertZettel()
	simpleZettel := newSimpleZettel()
	roFalse := newRoFalseZettel()
	roTrue := newRoTrueZettel()
	roReader := newRoReaderZettel()
	roWriter := newRoWriterZettel()
	roOwner := newRoOwnerZettel()

	testCases := []struct {
		user *meta.Meta
		meta *meta.Meta
		exp  bool
	}{
		// No meta
		{anonUser, nil, false},
		{reader, nil, false},
		{writer, nil, false},
		{owner, nil, false},
		{owner2, nil, false},
		// Any zettel
		{anonUser, zettel, !withAuth && !readonly},
		{reader, zettel, !withAuth && !readonly},
		{writer, zettel, !withAuth && !readonly},
		{owner, zettel, !readonly},
		{owner2, zettel, !readonly},
		// Expert zettel
		{anonUser, expertZettel, !withAuth && !readonly && expert},
		{reader, expertZettel, !withAuth && !readonly && expert},
		{writer, expertZettel, !withAuth && !readonly && expert},
		{owner, expertZettel, !readonly && expert},
		{owner2, expertZettel, !readonly && expert},
		// Simple expert zettel
		{anonUser, simpleZettel, !withAuth && !readonly && (simple || expert)},
		{reader, simpleZettel, !withAuth && !readonly && (simple || expert)},
		{writer, simpleZettel, !withAuth && !readonly && (simple || expert)},
		{owner, simpleZettel, !readonly && ((!withAuth && simple) || expert)},
		{owner2, simpleZettel, !readonly && ((!withAuth && simple) || expert)},
		// No r/o zettel
		{anonUser, roFalse, !withAuth && !readonly},
		{reader, roFalse, !withAuth && !readonly},
		{writer, roFalse, !withAuth && !readonly},
		{owner, roFalse, !readonly},
		{owner2, roFalse, !readonly},
		// Reader r/o zettel
		{anonUser, roReader, false},
		{reader, roReader, false},
		{writer, roReader, !withAuth && !readonly},
		{owner, roReader, !readonly},
		{owner2, roReader, !readonly},
		// Writer r/o zettel
		{anonUser, roWriter, false},
		{reader, roWriter, false},
		{writer, roWriter, false},
		{owner, roWriter, !readonly},

			if tc.exp != got {
				tt.Errorf("exp=%v, but got=%v", tc.exp, got)
			}
		})
	}
}

func testDelete(t *testing.T, pol Policy, simple, withAuth, readonly, expert bool) {
	t.Helper()
	anonUser := newAnon()
	reader := newReader()
	writer := newWriter()
	owner := newOwner()
	owner2 := newOwner2()
	zettel := newZettel()
	expertZettel := newExpertZettel()
	simpleZettel := newSimpleZettel()
	roFalse := newRoFalseZettel()
	roTrue := newRoTrueZettel()
	roReader := newRoReaderZettel()
	roWriter := newRoWriterZettel()
	roOwner := newRoOwnerZettel()

	testCases := []struct {
		user *meta.Meta
		meta *meta.Meta
		exp  bool
	}{
		// No meta
		{anonUser, nil, false},
		{reader, nil, false},
		{writer, nil, false},
		{owner, nil, false},
		{owner2, nil, false},
		// Any zettel
		{anonUser, zettel, !withAuth && !readonly},
		{reader, zettel, !withAuth && !readonly},
		{writer, zettel, !withAuth && !readonly},
		{owner, zettel, !readonly},
		{owner2, zettel, !readonly},
		// Expert zettel
		{anonUser, expertZettel, !withAuth && !readonly && expert},
		{reader, expertZettel, !withAuth && !readonly && expert},
		{writer, expertZettel, !withAuth && !readonly && expert},
		{owner, expertZettel, !readonly && expert},
		{owner2, expertZettel, !readonly && expert},
		// Simple expert zettel
		{anonUser, simpleZettel, !withAuth && !readonly && (simple || expert)},
		{reader, simpleZettel, !withAuth && !readonly && (simple || expert)},
		{writer, simpleZettel, !withAuth && !readonly && (simple || expert)},
		{owner, simpleZettel, !readonly && ((!withAuth && simple) || expert)},
		{owner2, simpleZettel, !readonly && ((!withAuth && simple) || expert)},
		// No r/o zettel
		{anonUser, roFalse, !withAuth && !readonly},
		{reader, roFalse, !withAuth && !readonly},
		{writer, roFalse, !withAuth && !readonly},
		{owner, roFalse, !readonly},
		{owner2, roFalse, !readonly},
		// Reader r/o zettel
		{anonUser, roReader, false},
		{reader, roReader, false},
		{writer, roReader, !withAuth && !readonly},
		{owner, roReader, !readonly},
		{owner2, roReader, !readonly},
		// Writer r/o zettel
		{anonUser, roWriter, false},
		{reader, roWriter, false},
		{writer, roWriter, false},
		{owner, roWriter, !readonly},

}
func newExpertZettel() *meta.Meta {
	m := meta.New(visZid)
	m.Set(meta.KeyTitle, "Expert Zettel")
	m.Set(meta.KeyVisibility, meta.ValueVisibilityExpert)
	return m
}
func newSimpleZettel() *meta.Meta {
	m := meta.New(visZid)
	m.Set(meta.KeyTitle, "Simple Zettel")
	m.Set(meta.KeyVisibility, meta.ValueVisibilitySimple)
	return m
}
func newRoFalseZettel() *meta.Meta {
	m := meta.New(zettelZid)
	m.Set(meta.KeyTitle, "No r/o Zettel")
	m.Set(meta.KeyReadOnly, "false")
	return m
}
func newRoTrueZettel() *meta.Meta {

//-----------------------------------------------------------------------------
// Copyright (c) 2020-2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package token provides some function for handling auth token.
package token

import (
	"errors"
	"time"

	"github.com/pascaldekloe/jwt"

	"zettelstore.de/z/config/startup"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
)

const reqHash = jwt.HS512

// ErrNoUser signals that the meta data has no role value 'user'.
var ErrNoUser = errors.New("auth: meta is no user")

// ErrNoIdent signals that the 'ident' key is missing.
var ErrNoIdent = errors.New("auth: missing ident")

// ErrOtherKind signals that the token was defined for another token kind.
var ErrOtherKind = errors.New("auth: wrong token kind")

// ErrNoZid signals that the 'zid' key is missing.
var ErrNoZid = errors.New("auth: missing zettel id")

// Kind specifies for which application / usage a token is/was requested.
type Kind int

// Allowed values of token kind
const (
	_ Kind = iota
	KindJSON
	KindHTML
)

// GetToken returns a token to be used for authentification.
func GetToken(ident *meta.Meta, d time.Duration, kind Kind) ([]byte, error) {
	if role, ok := ident.Get(meta.KeyRole); !ok || role != meta.ValueRoleUser {
		return nil, ErrNoUser
	}
	subject, ok := ident.Get(meta.KeyUserID)
	if !ok || subject == "" {
		return nil, ErrNoIdent
	}

	now := time.Now().Round(time.Second)
	claims := jwt.Claims{
		Registered: jwt.Registered{
			Subject: subject,
			Expires: jwt.NewNumericTime(now.Add(d)),
			Issued:  jwt.NewNumericTime(now),
		},
		Set: map[string]interface{}{
			"zid": ident.Zid.String(),
			"_tk": int(kind),
		},
	}
	token, err := claims.HMACSign(reqHash, startup.Secret())
	if err != nil {
		return nil, err
	}
	return token, nil
}

// ErrTokenExpired signals an exired token
var ErrTokenExpired = errors.New("auth: token expired")

// Data contains some important elements from a token.
type Data struct {
	Token   []byte
	Now     time.Time
	Issued  time.Time
	Expires time.Time
	Ident   string
	Zid     id.Zid
}

// CheckToken checks the validity of the token and returns relevant data.
func CheckToken(token []byte, k Kind) (Data, error) {
	h, err := jwt.NewHMAC(reqHash, startup.Secret())
	if err != nil {
		return Data{}, err
	}
	claims, err := h.Check(token)
	if err != nil {
		return Data{}, err
	}
	now := time.Now().Round(time.Second)
	expires := claims.Expires.Time()
	if expires.Before(now) {
		return Data{}, ErrTokenExpired
	}
	ident := claims.Subject
	if ident == "" {
		return Data{}, ErrNoIdent
	}
	if zidS, ok := claims.Set["zid"].(string); ok {
		if zid, err := id.Parse(zidS); err == nil {
			if kind, ok := claims.Set["_tk"].(float64); ok {
				if Kind(kind) == k {
					return Data{
						Token:   token,
						Now:     now,
						Issued:  claims.Issued.Time(),
						Expires: expires,
						Ident:   ident,
						Zid:     zid,
					}, nil
				}
			}
			return Data{}, ErrOtherKind
		}
	}
	return Data{}, ErrNoZid
}

//-----------------------------------------------------------------------------
// Copyright (c) 2020 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

package cmd

import (
	"flag"
	"fmt"

	"zettelstore.de/z/config/startup"
)

// ---------- Subcommand: config ---------------------------------------------

func cmdConfig(*flag.FlagSet) (int, error) {
	fmtVersion()
	fmt.Println("Stores")
	fmt.Printf("  Read-only mode    = %v\n", startup.IsReadOnlyMode())
	fmt.Println("Web")
	fmt.Printf("  Listen address    = %q\n", startup.ListenAddress())
	fmt.Printf("  URL prefix        = %q\n", startup.URLPrefix())
	if startup.WithAuth() {
		fmt.Println("Auth")
		fmt.Printf("  Owner             = %v\n", startup.Owner())
		fmt.Printf("  Secure cookie     = %v\n", startup.SecureCookie())
		fmt.Printf("  Persistent cookie = %v\n", startup.PersistentCookie())
		htmlLifetime, apiLifetime := startup.TokenLifetime()
		fmt.Printf("  HTML lifetime     = %v\n", htmlLifetime)
		fmt.Printf("  API lifetime      = %v\n", apiLifetime)
	}

	return 0, nil
}

import (
	"flag"
	"fmt"
	"io"
	"os"

	"zettelstore.de/z/config/runtime"
	"zettelstore.de/z/domain"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/encoder"
	"zettelstore.de/z/input"
	"zettelstore.de/z/parser"
)

// ---------- Subcommand: file -----------------------------------------------

func cmdFile(fs *flag.FlagSet) (int, error) {
	format := fs.Lookup("t").Value.String()
	meta, inp, err := getInput(fs.Args())
	if meta == nil {
		return 2, err
	}
	z := parser.ParseZettel(
		domain.Zettel{
			Meta:    meta,
			Content: domain.NewContent(inp.Src[inp.Pos:]),
		},
		runtime.GetSyntax(meta),

	)
	enc := encoder.Create(format, &encoder.Environment{Lang: runtime.GetLang(meta)})
	if enc == nil {
		fmt.Fprintf(os.Stderr, "Unknown format %q\n", format)
		return 2, nil
	}
	_, err = enc.WriteZettel(os.Stdout, z, format != "raw")
	if err != nil {
		return 2, err

	"zettelstore.de/z/auth/cred"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
)

// ---------- Subcommand: password -------------------------------------------

func cmdPassword(fs *flag.FlagSet) (int, error) {
	if fs.NArg() == 0 {
		fmt.Fprintln(os.Stderr, "User name and user zettel identification missing")
		return 2, nil
	}
	if fs.NArg() == 1 {
		fmt.Fprintln(os.Stderr, "User zettel identification missing")
		return 2, nil

// under this license.
//-----------------------------------------------------------------------------

package cmd

import (
	"flag"
	"log"
	"net/http"

	"zettelstore.de/z/auth/policy"
	"zettelstore.de/z/config/runtime"

	"zettelstore.de/z/config/startup"
	"zettelstore.de/z/place"
	"zettelstore.de/z/usecase"
	"zettelstore.de/z/web/adapter"
	"zettelstore.de/z/web/adapter/api"
	"zettelstore.de/z/web/adapter/webui"
	"zettelstore.de/z/web/router"
	"zettelstore.de/z/web/server"
	"zettelstore.de/z/web/session"
)

// ---------- Subcommand: run ------------------------------------------------

func flgRun(fs *flag.FlagSet) {
	fs.String("c", defConfigfile, "configuration file")

	fs.Uint("p", 23123, "port number")
	fs.String("d", "", "zettel directory")
	fs.Bool("r", false, "system-wide read-only mode")
	fs.Bool("v", false, "verbose mode")
	fs.Bool("debug", false, "debug mode")
}

func enableDebug(fs *flag.FlagSet, srv *server.Server) {

	if dbg := fs.Lookup("debug"); dbg != nil && dbg.Value.String() == "true" {
		srv.SetDebug()
	}
}





func runFunc(fs *flag.FlagSet) (int, error) {
	listenAddr := startup.ListenAddress()
	readonlyMode := startup.IsReadOnlyMode()
	logBeforeRun(listenAddr, readonlyMode)
	handler := setupRouting(startup.PlaceManager(), readonlyMode)
	srv := server.New(listenAddr, handler)
	enableDebug(fs, srv)
	if err := srv.Run(); err != nil {
		return 1, err
	}
	return 0, nil
}

func logBeforeRun(listenAddr string, readonlyMode bool) {
	v := startup.GetVersion()
	log.Printf("%v %v (%v@%v/%v)", v.Prog, v.Build, v.GoVersion, v.Os, v.Arch)
	log.Println("Licensed under the latest version of the EUPL (European Union Public License)")
	log.Printf("Listening on %v", listenAddr)
	log.Printf("Zettel location [%v]", startup.PlaceManager().Location())
	if readonlyMode {
		log.Println("Read-only mode")
	}
}

func setupRouting(mgr place.Manager, readonlyMode bool) http.Handler {
	var up place.Place = mgr
	pp, pol := policy.PlaceWithPolicy(
		up, startup.IsSimple(), startup.WithAuth, readonlyMode, runtime.GetExpertMode,
		startup.IsOwner, runtime.GetVisibility)

	te := webui.NewTemplateEngine(mgr, pol)

	ucAuthenticate := usecase.NewAuthenticate(up)

	ucGetMeta := usecase.NewGetMeta(pp)
	ucGetZettel := usecase.NewGetZettel(pp)
	ucParseZettel := usecase.NewParseZettel(ucGetZettel)
	ucListMeta := usecase.NewListMeta(pp)
	ucListRoles := usecase.NewListRole(pp)
	ucListTags := usecase.NewListTags(pp)
	ucZettelContext := usecase.NewZettelContext(pp)

	router := router.NewRouter()
	router.Handle("/", webui.MakeGetRootHandler(te, pp))
	router.AddListRoute('a', http.MethodGet, webui.MakeGetLoginHandler(te))
	router.AddListRoute('a', http.MethodPost, adapter.MakePostLoginHandler(
		api.MakePostLoginHandlerAPI(ucAuthenticate),
		webui.MakePostLoginHandlerHTML(te, ucAuthenticate)))
	router.AddListRoute('a', http.MethodPut, api.MakeRenewAuthHandler())
	router.AddZettelRoute('a', http.MethodGet, webui.MakeGetLogoutHandler(te))
	if !readonlyMode {
		router.AddZettelRoute('b', http.MethodGet, webui.MakeGetRenameZettelHandler(
			te, ucGetMeta))
		router.AddZettelRoute('b', http.MethodPost, webui.MakePostRenameZettelHandler(
			te, usecase.NewRenameZettel(pp)))
		router.AddZettelRoute('c', http.MethodGet, webui.MakeGetCopyZettelHandler(
			te, ucGetZettel, usecase.NewCopyZettel()))
		router.AddZettelRoute('c', http.MethodPost, webui.MakePostCreateZettelHandler(
			te, usecase.NewCreateZettel(pp)))
		router.AddZettelRoute('d', http.MethodGet, webui.MakeGetDeleteZettelHandler(
			te, ucGetZettel))
		router.AddZettelRoute('d', http.MethodPost, webui.MakePostDeleteZettelHandler(
			te, usecase.NewDeleteZettel(pp)))
		router.AddZettelRoute('e', http.MethodGet, webui.MakeEditGetZettelHandler(
			te, ucGetZettel))
		router.AddZettelRoute('e', http.MethodPost, webui.MakeEditSetZettelHandler(
			te, usecase.NewUpdateZettel(pp)))
		router.AddZettelRoute('f', http.MethodGet, webui.MakeGetFolgeZettelHandler(
			te, ucGetZettel, usecase.NewFolgeZettel()))
		router.AddZettelRoute('f', http.MethodPost, webui.MakePostCreateZettelHandler(
			te, usecase.NewCreateZettel(pp)))
		router.AddZettelRoute('g', http.MethodGet, webui.MakeGetNewZettelHandler(
			te, ucGetZettel, usecase.NewNewZettel()))
		router.AddZettelRoute('g', http.MethodPost, webui.MakePostCreateZettelHandler(
			te, usecase.NewCreateZettel(pp)))
	}
	router.AddListRoute('f', http.MethodGet, webui.MakeSearchHandler(
		te, usecase.NewSearch(pp), ucGetMeta, ucGetZettel))
	router.AddListRoute('h', http.MethodGet, webui.MakeListHTMLMetaHandler(
		te, ucListMeta, ucListRoles, ucListTags))
	router.AddZettelRoute('h', http.MethodGet, webui.MakeGetHTMLZettelHandler(
		te, ucParseZettel, ucGetMeta))
	router.AddZettelRoute('i', http.MethodGet, webui.MakeGetInfoHandler(
		te, ucParseZettel, ucGetMeta))
	router.AddZettelRoute('j', http.MethodGet, webui.MakeZettelContextHandler(te, ucZettelContext))

	router.AddZettelRoute('l', http.MethodGet, api.MakeGetLinksHandler(ucParseZettel))
	router.AddZettelRoute('o', http.MethodGet, api.MakeGetOrderHandler(
		usecase.NewZettelOrder(pp, ucParseZettel)))
	router.AddListRoute('r', http.MethodGet, api.MakeListRoleHandler(ucListRoles))
	router.AddListRoute('t', http.MethodGet, api.MakeListTagsHandler(ucListTags))
	router.AddZettelRoute('y', http.MethodGet, api.MakeZettelContextHandler(ucZettelContext))
	router.AddListRoute('z', http.MethodGet, api.MakeListMetaHandler(
		usecase.NewListMeta(pp), ucGetMeta, ucParseZettel))
	router.AddZettelRoute('z', http.MethodGet, api.MakeGetZettelHandler(
		ucParseZettel, ucGetMeta))


	return session.NewHandler(router, usecase.NewGetUserByZid(up))

}

//-----------------------------------------------------------------------------

package cmd

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"zettelstore.de/z/config/startup"
	"zettelstore.de/z/web/server"
)

func flgSimpleRun(fs *flag.FlagSet) {
	fs.String("d", "", "zettel directory")
}

func runSimpleFunc(*flag.FlagSet) (int, error) {

	listenAddr := startup.ListenAddress()
	readonlyMode := startup.IsReadOnlyMode()
	logBeforeRun(listenAddr, readonlyMode)
	if idx := strings.LastIndexByte(listenAddr, ':'); idx >= 0 {
		log.Println()
		log.Println("--------------------------")
		log.Printf("Open your browser and enter the following URL:")
		log.Println()
		log.Printf("    http://localhost%v", listenAddr[idx:])

	}

	handler := setupRouting(startup.PlaceManager(), readonlyMode)
	srv := server.New(listenAddr, handler)
	if err := srv.Run(); err != nil {
		return 1, err
	}
	return 0, nil
}

// runSimple is called, when the user just starts the software via a double click
// or via a simple call ``./zettelstore`` on the command line.
func runSimple() {
	dir := "./zettel"
	if err := os.MkdirAll(dir, 0750); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create zettel directory %q (%s)\n", dir, err)
		os.Exit(1)
	}
	executeCommand("run-simple", "-d", dir)
}

//-----------------------------------------------------------------------------
// Copyright (c) 2020 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

package cmd

import (
	"flag"
	"sort"


)

// Command stores information about commands / sub-commands.
type Command struct {
	Name   string              // command name as it appears on the command line
	Func   CommandFunc         // function that executes a command
	Places bool                // if true then places will be set up
	Simple bool                // Should start in simple mode
	Flags  func(*flag.FlagSet) // function to set up flag.FlagSet
	flags  *flag.FlagSet       // flags that belong to the command

}

// CommandFunc is the function that executes the command.
// It accepts the parsed command line parameters.
// It returns the exit code and an error.
type CommandFunc func(*flag.FlagSet) (int, error)

// GetFlags return the flag.FlagSet defined for the command.
func (c *Command) GetFlags() *flag.FlagSet { return c.flags }

var commands = make(map[string]Command)

// RegisterCommand registers the given command.

//-----------------------------------------------------------------------------
// Copyright (c) 2020-2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

package cmd

import (
	"context"
	"flag"
	"fmt"
	"log"

	"os"

	"strings"

	"zettelstore.de/z/config/runtime"
	"zettelstore.de/z/config/startup"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/index"
	"zettelstore.de/z/index/indexer"
	"zettelstore.de/z/input"
	"zettelstore.de/z/place"
	"zettelstore.de/z/place/manager"
	"zettelstore.de/z/place/progplace"

)

const (
	defConfigfile = ".zscfg"
)

func init() {
	RegisterCommand(Command{
		Name: "help",
		Func: func(*flag.FlagSet) (int, error) {
			fmt.Println("Available commands:")
			for _, name := range List() {
				fmt.Printf("- %q\n", name)
			}
			return 0, nil
		},
	})
	RegisterCommand(Command{
		Name: "version",
		Func: func(*flag.FlagSet) (int, error) {
			fmtVersion()
			return 0, nil
		},
	})
	RegisterCommand(Command{
		Name:   "run",
		Func:   runFunc,
		Places: true,

		Flags:  flgRun,
	})
	RegisterCommand(Command{
		Name:   "run-simple",
		Func:   runSimpleFunc,
		Places: true,
		Simple: true,
		Flags:  flgSimpleRun,
	})
	RegisterCommand(Command{
		Name:  "config",
		Func:  cmdConfig,
		Flags: flgRun,
	})
	RegisterCommand(Command{
		Name: "file",
		Func: cmdFile,
		Flags: func(fs *flag.FlagSet) {
			fs.String("t", "html", "target output format")
		},
	})
	RegisterCommand(Command{
		Name: "password",
		Func: cmdPassword,
	})
}

func fmtVersion() {
	version := startup.GetVersion()
	fmt.Printf("%v (%v/%v) running on %v (%v/%v)\n",
		version.Prog, version.Build, version.GoVersion,
		version.Hostname, version.Os, version.Arch)
}

func getConfig(fs *flag.FlagSet) (cfg *meta.Meta) {
	var configFile string
	if configFlag := fs.Lookup("c"); configFlag != nil {
		configFile = configFlag.Value.String()
	} else {
		configFile = defConfigfile
	}
	content, err := os.ReadFile(configFile)
	if err != nil {
		cfg = meta.New(id.Invalid)
	} else {

		cfg = meta.NewFromInput(id.Invalid, input.NewInput(string(content)))
	}



	fs.Visit(func(flg *flag.Flag) {
		switch flg.Name {
		case "p":
			cfg.Set(startup.KeyListenAddress, "127.0.0.1:"+flg.Value.String())






		case "d":
			val := flg.Value.String()
			if strings.HasPrefix(val, "/") {
				val = "dir://" + val
			} else {
				val = "dir:" + val
			}
			cfg.Set(startup.KeyPlaceOneURI, val)
		case "r":
			cfg.Set(startup.KeyReadOnlyMode, flg.Value.String())
		case "v":
			cfg.Set(startup.KeyVerbose, flg.Value.String())
		}
	})
	return cfg
}

func setupOperations(cfg *meta.Meta, withPlaces, simple bool) error {
	var mgr place.Manager
	var idx index.Indexer
	if withPlaces {
		err := raiseFdLimit()
		if err != nil {

			log.Println("Raising some limitions did not work:", err)



			log.Println("Prepare to encounter errors. Most of them can be mitigated. See the manual for details")


			cfg.Set(startup.KeyDefaultDirPlaceType, startup.ValueDirPlaceTypeSimple)











		}
		startup.SetupStartupConfig(cfg)
		idx = indexer.New()
		filter := index.NewMetaFilter(idx)
		mgr, err = manager.New(getPlaces(cfg), cfg.GetBool(startup.KeyReadOnlyMode), filter)
		if err != nil {
			return err

		}
	} else {
		startup.SetupStartupConfig(cfg)

	}



	startup.SetupStartupService(mgr, idx, simple)
	if withPlaces {
		if err := mgr.Start(context.Background()); err != nil {
			fmt.Fprintln(os.Stderr, "Unable to start zettel place")


			return err
		}
		runtime.SetupConfiguration(mgr)
		progplace.Setup(cfg, mgr, idx)
		idx.Start(mgr)
	}
	return nil
}











func getPlaces(cfg *meta.Meta) []string {
	var result []string = nil
	for cnt := 1; ; cnt++ {
		key := fmt.Sprintf("place-%v-uri", cnt)
		uri, ok := cfg.Get(key)
		if !ok || uri == "" {
			if cnt > 1 {
				break

			}
			uri = "dir:./zettel"
		}
		result = append(result, uri)




	}
	return result
}

func cleanupOperations(withPlaces bool) error {

	if withPlaces {
		startup.Indexer().Stop()

		if err := startup.PlaceManager().Stop(context.Background()); err != nil {






			fmt.Fprintln(os.Stderr, "Unable to stop zettel place")
			return err
		}


	}








	return nil


}

func executeCommand(name string, args ...string) {
	command, ok := Get(name)
	if !ok {
		fmt.Fprintf(os.Stderr, "Unknown command %q\n", name)
		os.Exit(1)
	}
	fs := command.GetFlags()
	if err := fs.Parse(args); err != nil {
		fmt.Fprintf(os.Stderr, "%s: unable to parse flags: %v %v\n", name, args, err)
		os.Exit(1)
	}
	cfg := getConfig(fs)
	if err := setupOperations(cfg, command.Places, command.Simple); err != nil {
		fmt.Fprintf(os.Stderr, "%s: %v\n", name, err)
		os.Exit(2)
	}


	exitCode, err := command.Func(fs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %v\n", name, err)
	}
	if err := cleanupOperations(command.Places); err != nil {
		fmt.Fprintf(os.Stderr, "%s: %v\n", name, err)
	}
	if exitCode != 0 {
		os.Exit(exitCode)
	}
}

// Main is the real entrypoint of the zettelstore.
func Main(progName, buildVersion string) {

	startup.SetupVersion(progName, buildVersion)

	if len(os.Args) <= 1 {
		runSimple()
	} else {
		executeCommand(os.Args[1], os.Args[2:]...)



	}
}

import (
	_ "zettelstore.de/z/encoder/htmlenc"   // Allow to use HTML encoder.
	_ "zettelstore.de/z/encoder/jsonenc"   // Allow to use JSON encoder.
	_ "zettelstore.de/z/encoder/nativeenc" // Allow to use native encoder.
	_ "zettelstore.de/z/encoder/rawenc"    // Allow to use raw encoder.
	_ "zettelstore.de/z/encoder/textenc"   // Allow to use text encoder.
	_ "zettelstore.de/z/encoder/zmkenc"    // Allow to use zmk encoder.

	_ "zettelstore.de/z/parser/blob"       // Allow to use BLOB parser.
	_ "zettelstore.de/z/parser/markdown"   // Allow to use markdown parser.
	_ "zettelstore.de/z/parser/none"       // Allow to use none parser.
	_ "zettelstore.de/z/parser/plain"      // Allow to use plain parser.
	_ "zettelstore.de/z/parser/zettelmark" // Allow to use zettelmark parser.
	_ "zettelstore.de/z/place/constplace"  // Allow to use global internal place.
	_ "zettelstore.de/z/place/dirplace"    // Allow to use directory place.
	_ "zettelstore.de/z/place/fileplace"   // Allow to use file place.
	_ "zettelstore.de/z/place/memplace"    // Allow to use memory place.

)

// Package collect provides functions to collect items from a syntax tree.
package collect

import "zettelstore.de/z/ast"

// DivideReferences divides the given list of rederences into zettel, local, and external References.
func DivideReferences(all []*ast.Reference, duplicates bool) (zettel, local, external []*ast.Reference) {
	if len(all) == 0 {
		return nil, nil, nil
	}

	mapZettel := make(map[string]bool)
	mapLocal := make(map[string]bool)
	mapExternal := make(map[string]bool)
	for _, ref := range all {
		if ref.State == ast.RefStateSelf {
			continue
		}
		if ref.IsZettel() {
			zettel = appendRefToList(zettel, mapZettel, ref, duplicates)
		} else if ref.IsExternal() {
			external = appendRefToList(external, mapExternal, ref, duplicates)
		} else {
			local = appendRefToList(local, mapLocal, ref, duplicates)
		}
	}
	return zettel, local, external
}

func appendRefToList(
	reflist []*ast.Reference,
	refSet map[string]bool,
	ref *ast.Reference,
	duplicates bool,
) []*ast.Reference {
	if duplicates {
		reflist = append(reflist, ref)
	} else {
		s := ref.String()
		if _, ok := refSet[s]; !ok {
			reflist = append(reflist, ref)
			refSet[s] = true
		}
	}

	return reflist
}

//-----------------------------------------------------------------------------
// Copyright (c) 2020 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package runtime provides functions to retrieve runtime configuration data.
package runtime

import (
	"zettelstore.de/z/config/startup"
	"zettelstore.de/z/domain/meta"
)

var mapDefaultKeys = map[string]func() string{
	meta.KeyCopyright: GetDefaultCopyright,
	meta.KeyLang:      GetDefaultLang,
	meta.KeyLicense:   GetDefaultLicense,
	meta.KeyRole:      GetDefaultRole,
	meta.KeySyntax:    GetDefaultSyntax,
	meta.KeyTitle:     GetDefaultTitle,
}

// AddDefaultValues enriches the given meta data with its default values.
func AddDefaultValues(m *meta.Meta) *meta.Meta {
	result := m
	for k, f := range mapDefaultKeys {
		if _, ok := result.Get(k); !ok {
			if result == m {
				result = m.Clone()
			}
			if val := f(); len(val) > 0 || m.Type(k) == meta.TypeEmpty {
				result.Set(k, val)
			}
		}
	}
	return result
}

// GetTitle returns the value of the "title" key of the given meta. If there
// is no such value, GetDefaultTitle is returned.
func GetTitle(m *meta.Meta) string {
	if syntax, ok := m.Get(meta.KeyTitle); ok && len(syntax) > 0 {
		return syntax
	}
	return GetDefaultTitle()
}

// GetRole returns the value of the "role" key of the given meta. If there
// is no such value, GetDefaultRole is returned.
func GetRole(m *meta.Meta) string {
	if syntax, ok := m.Get(meta.KeyRole); ok && len(syntax) > 0 {
		return syntax
	}
	return GetDefaultRole()
}

// GetSyntax returns the value of the "syntax" key of the given meta. If there
// is no such value, GetDefaultSyntax is returned.
func GetSyntax(m *meta.Meta) string {
	if syntax, ok := m.Get(meta.KeySyntax); ok && len(syntax) > 0 {
		return syntax
	}
	return GetDefaultSyntax()
}

// GetLang returns the value of the "lang" key of the given meta. If there is
// no such value, GetDefaultLang is returned.
func GetLang(m *meta.Meta) string {
	if lang, ok := m.Get(meta.KeyLang); ok && len(lang) > 0 {
		return lang
	}
	return GetDefaultLang()
}

// GetVisibility returns the visibility value, or "login" if none is given.
func GetVisibility(m *meta.Meta) meta.Visibility {
	if val, ok := m.Get(meta.KeyVisibility); ok {
		if vis := meta.GetVisibility(val); vis != meta.VisibilityUnknown {
			return vis
		}
	}
	return GetDefaultVisibility()
}

// GetUserRole role returns the user role of the given user zettel.
func GetUserRole(user *meta.Meta) meta.UserRole {
	if user == nil {
		if startup.WithAuth() {
			return meta.UserRoleUnknown
		}
		return meta.UserRoleOwner
	}
	if startup.IsOwner(user.Zid) {
		return meta.UserRoleOwner
	}
	if val, ok := user.Get(meta.KeyUserRole); ok {
		if ur := meta.GetUserRole(val); ur != meta.UserRoleUnknown {
			return ur
		}
	}
	return meta.UserRoleReader
}

//-----------------------------------------------------------------------------
// Copyright (c) 2020-2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package runtime provides functions to retrieve runtime configuration data.
package runtime

import (
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/place"
	"zettelstore.de/z/place/stock"
)

// --- Configuration zettel --------------------------------------------------

var configStock stock.Stock

// SetupConfiguration enables the configuration data.
func SetupConfiguration(mgr place.Manager) {
	if configStock != nil {
		panic("configStock already set")
	}
	configStock = stock.NewStock(mgr)
	if err := configStock.Subscribe(id.ConfigurationZid); err != nil {
		panic(err)
	}
}

// getConfigurationMeta returns the meta data of the configuration zettel.
func getConfigurationMeta() *meta.Meta {
	if configStock == nil {
		panic("configStock not set")
	}
	return configStock.GetMeta(id.ConfigurationZid)
}

// GetDefaultTitle returns the current value of the "default-title" key.
func GetDefaultTitle() string {
	if configStock != nil {
		if config := getConfigurationMeta(); config != nil {
			if title, ok := config.Get(meta.KeyDefaultTitle); ok {
				return title
			}
		}
	}
	return "Untitled"
}

// GetDefaultSyntax returns the current value of the "default-syntax" key.
func GetDefaultSyntax() string {
	if configStock != nil {
		if config := getConfigurationMeta(); config != nil {
			if syntax, ok := config.Get(meta.KeyDefaultSyntax); ok {
				return syntax
			}
		}
	}
	return meta.ValueSyntaxZmk
}

// GetDefaultRole returns the current value of the "default-role" key.
func GetDefaultRole() string {
	if configStock != nil {
		if config := getConfigurationMeta(); config != nil {
			if role, ok := config.Get(meta.KeyDefaultRole); ok {
				return role
			}
		}
	}
	return meta.ValueRoleZettel
}

// GetDefaultLang returns the current value of the "default-lang" key.
func GetDefaultLang() string {
	if configStock != nil {
		if config := getConfigurationMeta(); config != nil {
			if lang, ok := config.Get(meta.KeyDefaultLang); ok {
				return lang
			}
		}
	}
	return "en"
}

// GetDefaultCopyright returns the current value of the "default-copyright" key.
func GetDefaultCopyright() string {
	if configStock != nil {
		if config := getConfigurationMeta(); config != nil {
			if copyright, ok := config.Get(meta.KeyDefaultCopyright); ok {
				return copyright
			}
		}
	}
	return ""
}

// GetDefaultLicense returns the current value of the "default-license" key.
func GetDefaultLicense() string {
	if configStock != nil {
		if config := getConfigurationMeta(); config != nil {
			if license, ok := config.Get(meta.KeyDefaultLicense); ok {
				return license
			}
		}
	}
	return ""
}

// GetExpertMode returns the current value of the "expert-mode" key
func GetExpertMode() bool {
	if config := getConfigurationMeta(); config != nil {
		if mode, ok := config.Get(meta.KeyExpertMode); ok {
			return meta.BoolValue(mode)
		}
	}
	return false
}

// GetSiteName returns the current value of the "site-name" key.
func GetSiteName() string {
	if config := getConfigurationMeta(); config != nil {
		if name, ok := config.Get(meta.KeySiteName); ok {
			return name
		}
	}
	return "Zettelstore"
}

// GetHomeZettel returns the value of the "home-zettel" key.
func GetHomeZettel() id.Zid {
	if config := getConfigurationMeta(); config != nil {
		if start, ok := config.Get(meta.KeyHomeZettel); ok {
			if startID, err := id.Parse(start); err == nil {
				return startID
			}
		}
	}
	return id.DefaultHomeZid
}

// GetDefaultVisibility returns the default value for zettel visibility.
func GetDefaultVisibility() meta.Visibility {
	if config := getConfigurationMeta(); config != nil {
		if value, ok := config.Get(meta.KeyDefaultVisibility); ok {
			if vis := meta.GetVisibility(value); vis != meta.VisibilityUnknown {
				return vis
			}
		}
	}
	return meta.VisibilityLogin
}

// GetYAMLHeader returns the current value of the "yaml-header" key.
func GetYAMLHeader() bool {
	if config := getConfigurationMeta(); config != nil {
		return config.GetBool(meta.KeyYAMLHeader)
	}
	return false
}

// GetZettelFileSyntax returns the current value of the "zettel-file-syntax" key.
func GetZettelFileSyntax() []string {
	if config := getConfigurationMeta(); config != nil {
		return config.GetListOrNil(meta.KeyZettelFileSyntax)
	}
	return nil
}

// GetMarkerExternal returns the current value of the "marker-external" key.
func GetMarkerExternal() string {
	if config := getConfigurationMeta(); config != nil {
		if html, ok := config.Get(meta.KeyMarkerExternal); ok {
			return html
		}
	}
	return "➚"
}

// GetFooterHTML returns HTML code that should be embedded into the footer
// of each WebUI page.
func GetFooterHTML() string {
	if config := getConfigurationMeta(); config != nil {
		if data, ok := config.Get(meta.KeyFooterHTML); ok {
			return data
		}
	}
	return ""
}

// GetListPageSize returns the maximum length of a list to be returned in WebUI.
// A value less or equal to zero signals no limit.
func GetListPageSize() int {
	if config := getConfigurationMeta(); config != nil {
		if value, ok := config.GetNumber(meta.KeyListPageSize); ok && value > 0 {
			return value
		}
	}
	return 0
}

//-----------------------------------------------------------------------------
// Copyright (c) 2020-2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package startup provides functions to retrieve startup configuration data.
package startup

import (
	"hash/fnv"
	"io"
	"strconv"
	"time"

	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/index"
	"zettelstore.de/z/place"
)

var config struct {
	// Set in SetupStartupConfig
	verbose             bool
	readonlyMode        bool
	urlPrefix           string
	listenAddress       string
	defaultDirPlaceType string
	owner               id.Zid
	withAuth            bool
	secret              []byte
	insecCookie         bool
	persistCookie       bool
	htmlLifetime        time.Duration
	apiLifetime         time.Duration

	// Set in SetupStartupService
	simple  bool // was started without run command
	manager place.Manager
	indexer index.Indexer
}

// Predefined keys for startup zettel
const (
	KeyDefaultDirPlaceType = "default-dir-place-type"
	KeyInsecureCookie      = "insecure-cookie"
	KeyListenAddress       = "listen-addr"
	KeyOwner               = "owner"
	KeyPersistentCookie    = "persistent-cookie"
	KeyPlaceOneURI         = "place-1-uri"
	KeyReadOnlyMode        = "read-only-mode"
	KeyTokenLifetimeHTML   = "token-lifetime-html"
	KeyTokenLifetimeAPI    = "token-lifetime-api"
	KeyURLPrefix           = "url-prefix"
	KeyVerbose             = "verbose"
)

// Important values for some keys.
const (
	ValueDirPlaceTypeNotify = "notify"
	ValueDirPlaceTypeSimple = "simple"
)

// SetupStartupConfig initializes the startup data with content of config file.
func SetupStartupConfig(cfg *meta.Meta) {
	if config.urlPrefix != "" {
		panic("startup.config already set")
	}
	config.verbose = cfg.GetBool(KeyVerbose)
	config.readonlyMode = cfg.GetBool(KeyReadOnlyMode)
	config.urlPrefix = cfg.GetDefault(KeyURLPrefix, "/")
	if prefix, ok := cfg.Get(KeyURLPrefix); ok &&
		len(prefix) > 0 && prefix[0] == '/' && prefix[len(prefix)-1] == '/' {
		config.urlPrefix = prefix
	} else {
		config.urlPrefix = "/"
	}
	if val, ok := cfg.Get(KeyListenAddress); ok {
		config.listenAddress = val // TODO: check for valid string
	} else {
		config.listenAddress = "127.0.0.1:23123"
	}
	if defaultType, ok := cfg.Get(KeyDefaultDirPlaceType); ok {
		switch defaultType {
		case ValueDirPlaceTypeNotify:
		case ValueDirPlaceTypeSimple:
		default:
			defaultType = ValueDirPlaceTypeNotify
		}
		config.defaultDirPlaceType = defaultType
	} else {
		config.defaultDirPlaceType = ValueDirPlaceTypeNotify
	}
	config.owner = id.Invalid
	if owner, ok := cfg.Get(KeyOwner); ok {
		if zid, err := id.Parse(owner); err == nil {
			config.owner = zid
			config.withAuth = true
		}
	}
	if config.withAuth {
		config.insecCookie = cfg.GetBool(KeyInsecureCookie)
		config.persistCookie = cfg.GetBool(KeyPersistentCookie)
		config.secret = calcSecret(cfg)
		config.htmlLifetime = getDuration(
			cfg, KeyTokenLifetimeHTML, 1*time.Hour, 1*time.Minute, 30*24*time.Hour)
		config.apiLifetime = getDuration(
			cfg, KeyTokenLifetimeAPI, 10*time.Minute, 0, 1*time.Hour)
	}
}

// SetupStartupService initializes the startup data with internal services.
func SetupStartupService(manager place.Manager, idx index.Indexer, simple bool) {
	if config.urlPrefix == "" {
		panic("startup.config not set")
	}
	config.simple = simple && !config.withAuth
	config.manager = manager
	config.indexer = idx
}

func calcSecret(cfg *meta.Meta) []byte {
	h := fnv.New128()
	if secret, ok := cfg.Get("secret"); ok {
		io.WriteString(h, secret)
	}
	io.WriteString(h, version.Prog)
	io.WriteString(h, version.Build)
	io.WriteString(h, version.Hostname)
	io.WriteString(h, version.Os)
	io.WriteString(h, version.Arch)
	return h.Sum(nil)
}

func getDuration(
	cfg *meta.Meta, key string, defDur, minDur, maxDur time.Duration) time.Duration {
	if s, ok := cfg.Get(key); ok && len(s) > 0 {
		if d, err := strconv.ParseUint(s, 10, 64); err == nil {
			secs := time.Duration(d) * time.Minute
			if secs < minDur {
				return minDur
			}
			if secs > maxDur {
				return maxDur
			}
			return secs
		}
	}
	return defDur
}

// IsSimple returns true if Zettelstore was not started with command "run"
// and authentication is disabled.
func IsSimple() bool { return config.simple }

// IsVerbose returns whether the system should be more chatty about its operations.
func IsVerbose() bool { return config.verbose }

// IsReadOnlyMode returns whether the system is in read-only mode or not.
func IsReadOnlyMode() bool { return config.readonlyMode }

// URLPrefix returns the configured prefix to be used when providing URL to
// the service.
func URLPrefix() string { return config.urlPrefix }

// ListenAddress returns the string that specifies the the network card and the ip port
// where the server listens for requests
func ListenAddress() string { return config.listenAddress }

// DefaultDirPlaceType returns the default value for a directory place type.
func DefaultDirPlaceType() string { return config.defaultDirPlaceType }

// WithAuth returns true if user authentication is enabled.
func WithAuth() bool { return config.withAuth }

// SecureCookie returns whether the web app should set cookies to secure mode.
func SecureCookie() bool { return config.withAuth && !config.insecCookie }

// PersistentCookie returns whether the web app should set persistent cookies
// (instead of temporary).
func PersistentCookie() bool { return config.persistCookie }

// Owner returns the zid of the zettelkasten's owner.
// If there is no owner defined, the value ZettelID(0) is returned.
func Owner() id.Zid { return config.owner }

// IsOwner returns true, if the given user is the owner of the Zettelstore.
func IsOwner(zid id.Zid) bool { return zid.IsValid() && zid == config.owner }

// Secret returns the interal application secret. It is typically used to
// encrypt session values.
func Secret() []byte { return config.secret }

// TokenLifetime return the token lifetime for the web/HTML access and for the
// API access. If lifetime for API access is equal to zero, no API access is
// possible.
func TokenLifetime() (htmlLifetime, apiLifetime time.Duration) {
	return config.htmlLifetime, config.apiLifetime
}

// PlaceManager returns the managing place.
func PlaceManager() place.Manager { return config.manager }

// Indexer returns the current indexer.
func Indexer() index.Indexer { return config.indexer }

//-----------------------------------------------------------------------------
// Copyright (c) 2020 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package startup provides functions to retrieve startup configuration data.
package startup

import (
	"os"
	"runtime"
)

// Version describes all elements of a software version.
type Version struct {
	Prog      string // Name of the software
	Build     string // Representation of build process
	Hostname  string // Host name a reported by the kernel
	GoVersion string // Version of go
	Os        string // GOOS
	Arch      string // GOARCH
	// More to come
}

var version Version

// SetupVersion initializes the version data.
func SetupVersion(progName, buildVersion string) {
	version.Prog = progName
	if buildVersion == "" {
		version.Build = "unknown"
	} else {
		version.Build = buildVersion
	}
	if hn, err := os.Hostname(); err == nil {
		version.Hostname = hn
	} else {
		version.Hostname = "*unknown host*"
	}
	version.GoVersion = runtime.Version()
	version.Os = runtime.GOOS
	version.Arch = runtime.GOARCH
}

// GetVersion returns the current software version data.
func GetVersion() Version { return version }

id: 00001004000000
title: Configuration of Zettelstore
role: manual
tags: #configuration #manual #zettelstore
syntax: zmk


There are two levels to change the behavior and/or the appearance of Zettelstore.
The first level is the configuration that is needed to start the services provided by Zettelstore.



For example, this includes the URI under which your Zettelstore is accessible.

* [[Zettelstore start-up configuration|00001004010000]]

The second level is configuring the running Zettelstore.
For example, you can configure the default language of your Zettelstore.
* [[Configure a running Zettelstore|00001004020000]]

The third level is the way to start Zettelstore services and to manage it.
* [[Command line parameters|00001004050000]]

id: 00001004010000
title: Zettelstore start-up configuration
role: manual
tags: #configuration #manual #zettelstore
syntax: zmk


The configuration file, as specified by the ''-c CONFIGFILE'' [[command line option|00001004051000]], allows you to specify some start-up options.
These options cannot be stored in a [[configuration zettel|00001004020000]] because either they are needed before Zettelstore can start or because of security reasons.
For example, Zettelstore need to know in advance, on which network address is must listen or where zettel are placed.
An attacker that is able to change the owner can do anything.
Therefore only the owner of the computer on which Zettelstore runs can change this information.

The file for start-up configuration must be created via a text editor in advance.

The syntax of the configuration file is the same as for any zettel metadata.
The following keys are supported:








; [!default-dir-place-type]''default-dir-place-type''
: Specifies the default value for the (sub-) type of [[directory places|00001004011400#type]].
  Zettel are typically stored in such places.

  Default: ''notify''
; [!insecure-cookie]''insecure-cookie''
: Must be set to ''true'', if authentication is enabled and Zettelstore is not accessible not via HTTPS (but via HTTP).

  On these devices, the operating system is free to stop the web browser and to remove temporary cookies.
  Therefore, an authenticated user will be logged off.

  If ''true'', a persistent cookie is used.
  Its lifetime exceeds the lifetime of the authentication token (see option ''token-lifetime-html'') by 30 seconds.

  Default: ''false''
; [!place-X-uri]''place-//X//-uri'', where //X// is a number greater or equal to one
: Specifies a [[place|00001004011200]] where zettel are stored.
  During start-up //X// is counted, starting with one, until no key is found.
  This allows to configure more than one place.

  If no ''place-1-uri'' key is given, the overall effect will be the same as if only ''place-1-uri'' was specified with the value ''dir://.zettel''.
  In this case, even a key ''place-2-uri'' will be ignored.
; [!read-only-mode]''read-only-mode''
: Puts the Zettelstore web service into a read-only mode.
  No changes are possible.
  Default: false.
; [!token-lifetime-api]''token-lifetime-api'', [!token-lifetime-html]''token-lifetime-html''
: Define lifetime of access tokens in minutes.
  Values are only valid if authentication is enabled, i.e. key ''owner'' is set.

id: 00001004011200
title: Zettelstore places
role: manual
tags: #configuration #manual #zettelstore
syntax: zmk


A Zettelstore must store its zettel somehow and somewhere.
In most cases you want to store your zettel as files in a directory.
Under certain circumstances you may want to store your zettel in other places.

An example are the [[predefined zettel|00001005090000]] that come with a Zettelstore.
They are stored within the software itself.
In another situation you may want to store your zettel volatile, e.g. if you want to provide a sandbox for experimenting.

To cope with these (and more) situations, you configure Zettelstore to use one or more places.
This is done via the ''place-X-uri'' keys of the [[start-up configuration|00001004010000#place-X-uri]] (X is a number).
Places are specified using special [[URIs|https://en.wikipedia.org/wiki/Uniform_Resource_Identifier]], somehow similar to web addresses.

The following place URIs are supported:

; ''dir:\//DIR''
: Specifies a directory where zettel files are stored.
  ''DIR'' is the file path.

  You can create such a ZIP file, if you zip a directory full of zettel files.

  This place is always read-only.
; ''mem:''
: Stores all its zettel in volatile memory.
  If you stop the Zettelstore, all changes are lost.

All places that you configure via the ''store-X-uri'' keys form a chain of places.
If a zettel should be retrieved, a search starts in the place specified with the ''place-1-uri'' key, then ''place-2-uri'' and so on.
If a zettel is created or changed, it is always stored in the place specified with the ''place-1-uri'' key.
This allows to overwrite zettel from other places, e.g. the predefined zettel.

If you did not configure the place of the predefined zettel (''const:'') they will automatically be appended as a last place.
Otherwise Zettelstore will not work in certain situations.

If you use the ''mem:'' place, where zettel are stored in volatile memory, it makes only sense if you configure it as ''place-1-uri''.
Such a place will be empty when Zettelstore starts and only the place 1 will receive updates.
You must make sure that your computer has enough RAM to store all zettel.

id: 00001004011400
title: Configure file directory places
role: manual
tags: #configuration #manual #zettelstore
syntax: zmk


Under certain circumstances, it is preferable to further configure a file directory place.
This is done by appending query parameters after the base place URI ''dir:\//DIR''.

The following parameters are supported:

|= Parameter:|Description|Default value:|

When the parameter ''type'' is set to ""notify"", Zettelstore automatically detects changes to zettel files that originates from other software.
It is done on a ""best-effort"" basis.
Under certain circumstances it is possible that Zettelstore does not detect a change done by another software.

To cope with this unlikely, but still possible event, Zettelstore re-scans periodically the file directory.
The time interval is configured by the ''rescan'' parameter, e.g.
```
place-1-uri: dir:///home/zettel?rescan=300
```
This makes Zettelstore to re-scan the directory ''/home/zettel/'' every 300 seconds, i.e. 5 minutes.

For a Zettelstore with many zettel, re-scanning the directory may take a while, especially if it is stored on a remote computer (e.g. via CIFS/SMB or NFS).
In this case, you should adjust the parameter value.

Please note that a directory re-scan invalidates all internal data of a Zettelstore.

For various reasons, the value should be a prime number, with a maximum value of 1499.

=== Readonly
Sometimes you may want to provide zettel from a file directory place, but you want to disallow any changes.
If you provide the query parameter ''readonly'' (with or without a corresponding value), the place will disallow any changes.
```
place-1-uri: dir:///home/zettel?readonly
```
If you put the whole Zettelstore in [[read-only|00001004010000]] [[mode|00001004051000]], all configured file directory places will be in read-only mode too, even if not explicitly configured.

id: 00001004050000
title: Command line parameters
role: manual
tags: #command #configuration #manual #zettelstore
syntax: zmk


Zettelstore is not just a web service that provides services of a zettelkasten.
It allows to some tasks to be executed at the command line.
Typically, the task (""sub-command"") will be given at the command line as the first parameter.

If no parameter is given, the Zettelstore is called as
```
zettelstore
```
This is equivalent to call it this way:
```sh
mkdir -p ./zettel
zettelstore run -d ./zettel -c ./.zscfg
```
Typically this is done by starting Zettelstore via a graphical user interface by double-clicking to its file icon.
=== Sub-commands
* [[``zettelstore help``|00001004050200]] lists all available sub-commands.
* [[``zettelstore version``|00001004050400]] to display version information of Zettelstore.
* [[``zettelstore config``|00001004050600]] to show the currently active [[configuration|00001004000000]].
* [[``zettelstore run``|00001004051000]] to start the web-based Zettelstore service.
* [[``zettelstore run-simple``|00001004051100]] is typically called, when you start Zettelstore by a double.click in your GUI.
* [[``zettelstore file``|00001004051200]] to render files manually without activated/running Zettelstore services.
* [[``zettelstore password``|00001004051400]] to calculate data for user authentication.

id: 00001004050600
title: The ''config'' sub-command
role: manual
tags: #command #configuration #manual #zettelstore
syntax: zmk
precursor: 00001004050000

Shows the Zettelstore configuration, for debugging purposes.
Currently, only the [[start-up configuration|00001004010000]] is shown.

This sub-command uses the same command line parameters as [[``zettelstore run``|00001004051000]].

An example for an unconfigured Zettelstore:
```
# zettelstore config
Zettelstore (v0.0.4/go1.15) running on mycomputer (linux/amd64)
Stores
  Read only         = false
Web
  Listen Addr       = "127.0.0.1:23123"
  URL prefix        = "/"
```
The first line is identical to the output of the [[``zettelkasten version``|00001004050400]] sub-command.

id: 00001004051000
title: The ''run'' sub-command
role: manual
tags: #command #configuration #manual #zettelstore
syntax: zmk

precursor: 00001004050000

=== ``zettelstore run``
This starts the web service.

```
zettelstore run [-c CONFIGFILE] [-d DIR] [-p PORT] [-r] [-v]
```




; ''-c CONFIGFILE''
: Specifies ''CONFIGFILE'' as a file, where [[start-up configuration data|00001004010000]] is read.
  It is ignored, when the given file is not available, nor readable.

  Default: ''./.zscfg''. (''.\\.zscfg'' on Windows)), where ''.'' denotes the ""current directory"".
; ''-d DIR''
: Specifies ''DIR'' as the directory that contains all zettel.

  Default is ''./zettel'' (''.\\zettel'' on Windows), where ''.'' denotes the ""current directory"".
; [!debug]''-debug''
: Allows better debugging of the internal web server by disabling any timeout values.
  You should specify this only as a developer.
  Especially do not enable it for a production server.

  [[https://blog.cloudflare.com/exposing-go-on-the-internet/#timeouts]] contains a good explanation for the usefulness of sensitive timeout values.
; ''-p PORT''
: Specifies the integer value ''PORT'' as the TCP port, where the Zettelstore service listens for requests.

  Default: 23123.

  Zettelstore listens only on ''127.0.0.1'', e.g. only requests from the current computer will be processed.
  If you want to listen on network card to process requests from other computer, please use ''listen-addr'' of the configuration file as described below.
; ''-r''
: Puts the Zettelstore in read-only mode.
  No changes are possible via the web interface / via the API.

  This allows to publish your content without any risks of unauthorized changes.
; ''-v''
: Be more verbose in writing logs.
  Writes the start-up configuration to stderr.

Command line options take precedence over configuration file options.

Zettelstore becomes extensible by external software.
For example, a more sophisticated web interface could be build, or an application for your mobile device that allows you to send content to your Zettelstore as new zettel.

=== Where zettel are stored

Your zettel are stored as files in a specific directory.
If you have not explicitly specified the directory, a default directory will be used.
The directory has to be specified at [[start-up time|00001004010000]].
Nested directories are not supported (yet).

Every file in this directory that should be monitored by Zettelstore must have a file name that begins with 14 digits (0-9), the [[zettel identifier|00001006050000]].
If you create a new zettel via the web interface or the API, the zettel identifier will be the timestamp of the current date and time (format is ''YYYYMMDDhhmmss'').
This allows zettel to be sorted naturally by creation time.

Since the only restriction on zettel identifiers are the 14 digits, you are free to use other digit sequences.

id: 00001005090000
title: List of predefined zettel
role: manual
tags: #manual #reference #zettelstore
syntax: zmk


The following table lists all predefined zettel with their purpose.

|= Identifier :|= Title | Purpose
| [[00000000000001]] | Zettelstore Version | Contains the version string of the running Zettelstore
| [[00000000000002]] | Zettelstore Host | Contains the name of the computer running the Zettelstore
| [[00000000000003]] | Zettelstore Operating System | Contains the operating system and CPU architecture of the computer running the Zettelstore
| [[00000000000006]] | Zettelstore Environment Values | Contains environmental data of Zettelstore executable
| [[00000000000008]] | Zettelstore Runtime Values | Contains values that reflect the inner working; see [[here|https://golang.org/pkg/runtime/]] for a technical description of these values
| [[00000000000018]] | Zettelstore Indexer | Provides some statistics about the index process
| [[00000000000020]] | Zettelstore Place Manager | Contains some statistics about zettel places
| [[00000000000090]] | Zettelstore Supported Metadata Keys | Contains all supported metadata keys, their [[types|00001006030000]], and more
| [[00000000000096]] | Zettelstore Start-up Configuration | Contains the effective values of the [[start-up configuration|00001004010000]]
| [[00000000000098]] | Zettelstore Start-up Values | Contains all values computed from the [[start-up configuration|00001004010000]]
| [[00000000000100]] | Zettelstore Runtime Configuration | Allows to [[configure Zettelstore at runtime|00001004020000]]
| [[00000000010100]] | Zettelstore Base HTML Template | Contains the general layout of the HTML view
| [[00000000010200]] | Zettelstore Login Form HTML Template | Layout of the login form, when authentication is [[enabled|00001010040100]]
| [[00000000010300]] | Zettelstore List Meta HTML Template | Used when displaying a list of zettel
| [[00000000010401]] | Zettelstore Detail HTML Template | Layout for the HTML detail view of one zettel
| [[00000000010402]] | Zettelstore Info HTML Templöate | Layout for the information view of a specific zettel
| [[00000000010403]] | Zettelstore Form HTML Template | Form that is used to create a new or to change an existing zettel that contains text
| [[00000000010404]] | Zettelstore Rename Form HTML Template | View that is displayed to change the [[zettel identifier|00001006050000]]
| [[00000000010405]] | Zettelstore Delete HTML Template | View to confirm the deletion of a zettel
| [[00000000010500]] | Zettelstore List Roles HTML Template | Layout for listing all roles
| [[00000000010600]] | Zettelstore List Tags HTML Template | Layout of tags lists
| [[00000000020001]] | Zettelstore Base CSS | CSS file that is included by the [[Base HTML Template|00000000010100]]

| [[00000000090000]] | New Menu | Contains items that should contain in the zettel template menu
| [[00000000090001]] | New Zettel | Template for a new zettel with role ""[[zettel|00001006020100]]""
| [[00000000090002]] | New User | Template for a new zettel with role ""[[user|00001006020100#user]]""
| [[00010000000000]] | Home | Default home zettel, contains some welcome information

If a zettel is not linked, it is not accessible for the current user.

**Important:** The identifier may change until a stable version of the software is released.

id: 00001006034500
title: Timestamp Key Type
role: manual
tags: #manual #meta #reference #zettel #zettelstore
syntax: zmk


Values of this type denote a point in time.

=== Allowed values
Must be a sequence of 14 digits (""0""--""9"") (same as an [[Identifier|00001006032000]]), with the restriction that is conforms to the pattern ""YYYYMMDDhhmmss"".

* YYYY is the year,
* MM is the month,
* DD is the day,
* hh is the hour,
* mm is the minute,
* ss is the second.

=== Match operator
A value matches an timestampvalue, if the first value is the prefix of the timestamp value.

For example, ""202102"" matches ""20210212143200"".

=== Sorting
Sorting is done by comparing the [[String|00001006033500]] values.

If both values are timestamp values, this works well because both have the same length.

id: 00001007031000
title: Zettelmarkup: Tables

tags: #manual #zettelmarkup #zettelstore
syntax: zmk
role: manual

Tables are used to show some data in a two-dimenensional fashion.
In zettelmarkup, table are not specified explicitly, but by entering //table rows//.
Therefore, a table can be seen as a sequence of table rows.
A table row is nothing as a sequence of //table cells//.
The length of a table is the number of table rows, the width of a table is the maximum length of its rows.

will be rendered in HTML as:
:::example
| a1 | a2 | a3|
| b1 | b2 | b3
| c1 | c2
:::


If any cell in the first row of a table contains an equal sing character (""''=''"", ''U+003D'') as the very first character, then this first row will be interpreted as a //table header// row.

For example:
```zmk
| a1 | a2 |= a3|
| b1 | b2 | b3
| c1 | c2
```
will be rendered in HTML as:
:::example
| a1 | a2 |= a3|
| b1 | b2 | b3
| c1 | c2
:::


Inside a header row, you can specify the alignment of each header cell by a given character as the last character of a cell.
The alignment of a header cell determines the alignment of every cell in the same column.
The following characters specify the alignment:

* the colon character (""'':''"", ''U+003A'') forces a centered aligment,
* the less-than sign character (""''<''"", ''U+0060'') specifies an alignment to the left,
* the greater-than sign character (""''>''"", ''U+0062'') will produce right aligned cells.

will be rendered in HTML as:
:::example
|=Left<|Right>|Center:|Default
|123456|123456|123456|123456|
|123|123|123|123
:::


To specify the alignment of an individual cell, you can enter these characters for alignment as the first character of that cell.

For example:
```zmk
|=Left<|Right>|Center:|Default
|>R|:C|<L
|123456|123456|123456|123456|
|123|123|123|123
```
will be rendered in HTML as:
:::example
|=Left<|Right>|Center:|Default
|>R|:C|<L
|123456|123456|123456|123456|
|123|123|123|123
:::

id: 00001007040200
title: Zettelmarkup: Literal-like formatting

tags: #manual #zettelmarkup #zettelstore
syntax: zmk
role: manual

There are some reasons to mark text that should be rendered as uninterpreted:
* Mark text as literal, sometimes as part of a program.
* Mark text as input you give into a computer via a keyboard.
* Mark text as output from some computer, e.g. shown at the command line.

=== Literal text
Literal text somehow relates to [[verbatim blocks|00001007030500]]: their content should not be interpreted further, but may be rendered special.
It is specified by two grave accent characters (""''`''"", ''U+0060''), followed by the text, followed by again two grave accent characters, optionally followed by an [[attribute|00001007050000]] specification.
Similar to the verbatim block, the literal element allows also a modifier letter grave accent (""''ˋ''"", ''U+02CB'') as an alternative to the grave accent character[^On some devices, such as an iPhone / iPad, a grave accent character is harder to enter and is often confused with a modifier letter grave accent.].
However, all four characters must be the same.

The literal element supports the default attribute: when given, all spaces in the text are rendered in HTML as open box characters (""''␣''"", ''U+2423'').
The use of a generic attribute allwos to specify a (programming) language that controls syntax colouring when rendered in HTML.

If you want to specify a grave accent character in the text, either use modifier grave accent characters as delimiters for the element, or place a backslash character before the grave accent character you want to use inside the element.
If you want to enter a backslash character, you need to enter two of these.

Examples:
* ``\`\`abc def\`\``` is rendered in HTML as ::``abc def``::{=example}.
* ``\`\`abc def\`\`{-}`` is rendered in HTML as ::``abc def``{-}::{=example}.
* ``\`\`abc\\\`def\`\``` is rendered in HTML as ::``abc\`def``::{=example}.
* ``\`\`abc\\\\def\`\``` is rendered in HTML as ::``abc\\def``::{=example}.

If the text is given, it will be interpreted as an alternative textual representation, to help persons with some visual disabilities.

[[Attributes|00001007050000]] are supported.
They must follow the last right curly bracket character immediately.
One prominent example is to specify an explicit title attribute that is shown on certain web browsers when the zettel is rendered in HTML:

%%Example:


%%``{{External link|00000000030001}}{title=External width=30}`` is rendered as ::{{External link|00000000030001}}{title=External width=30}::{=example}.


=== Footnotes

A footnote begins with a left square bracket, followed by a circumflex accent (""''^''"", ''U+005E''), followed by some text, and ends with a right square bracket.

Example:

id: 00001008000000
title: Other Markup Languages
role: manual
tags: #manual #zettelstore
syntax: zmk


[[Zettelmarkup|00001007000000]] is not the only markup language you can use to define your content.
Zettelstore is quite agnostic with respect to markup languages.
Of course, Zettelmarkup plays an important role.
However, with the exception of zettel titles, you can use any (markup) language that is supported:

* Markdown
* Images: GIF, PNG, JPEG, SVG
* CSS
* HTML template data


* Plain text, not further interpreted

The [[metadata key|00001006020000#syntax]] ""''syntax''"" specifies which language should be used.
If it is not given, the key ""''default-syntax''"" will be used (specified in the [[configuration zettel|00001004020000#default-syntax]]).
The following syntax values are supported:

; [!css]''css''
: A [[Cascading Style Sheet|https://www.w3.org/Style/CSS/]], to be used when rendering a zettel as HTML.
; [!gif]''gif''; [!jpeg]''jpeg''; [!jpg]''jpg''; [!png]''png''
: The formats for pixel graphics.
  Typically the data is stored in a separate file and the syntax is given in the ''.meta'' file.
; [!markdown]''markdown'', [!md]''md''
: For those who desperately need [[Markdown|https://daringfireball.net/projects/markdown/]].
  Since the world of Markdown is so diverse, a [[CommonMark|https://commonmark.org]] parser is used.
  See [[Use Markdown as the main markup language of Zettelstore|00001008010000]].
; [!mustache]''mustache''
: A [[Mustache template|https://mustache.github.io/]], used when rendering a zettel as HTML.
; [!none]''none''
: Only the metadata of a zettel is ""parsed"".
  Useful for displaying the full metadata.
  The [[runtime configuration zettel|00000000000100]] uses this syntax.
  The zettel content is ignored.

id: 00001008010000
title: Use Markdown as the main markup language of Zettelstore

tags: #manual #markdown #zettelstore
syntax: zmk
role: manual

If you are customized to use Markdown as your markup language, you can configure Zettelstore to support your decision.




Just add the key ''default-syntax'' with a value of ''md'' or ''markdown'' to the [[configuration zettel|00000000000100]].
Whether to use ''md'' or ''markdown'' is not just a matter to taste, but also depends on the value of ''zettel-file-syntax'' and, to some degree, on the value of ''yaml-header''.
All key are described [[here|00001004020000]].

If you set ''yaml-header'' to true, then new content is always stored in a file with the extension ''.zettel''.

Otherwise ''zettel-file-syntax'' lists all syntax values, where its content should be stored in a file with the extension ''.zettel''.

If neither ''yaml-header'' nor ''zettel-file-syntax'' is set, new content is stored in a file where its file name extension is the same as the syntax value of that zettel.
In this case it makes a difference, whether you specify ''md'' or ''markdown''.
If you specify the syntax ''md'', your content will be stored in a file with the ''.md'' extension.
Similar for the syntax ''markdown''.

If you want to process the files that store the zettel content, e.g. with some other Markdown tools, this may be important.
Not every Markdown tool allows both file extensions.

BTW, metadata is stored in a file with the extension ''.meta'', if neither ''yaml-header'' nor ''zettel-file-syntax'' is set.

id: 00001010000000
title: Security

tags: #configuration #manual #security #zettelstore
syntax: zmk
role: manual

Your zettel could contain sensitive content.
You probably want to ensure that only authorized person can read and/or modify them.
Zettelstore ensures this in various ways.

=== Local first
The Zettelstore is designed to run on your local computer.
If you do not configure it in other ways, no person from another computer can connect to your Zettelstore.
You must explicitly configure it to allow access from other computers.

In the case that your own multiple computers, you do not have to access the Zettelstore remotely.
You could install Zettelstore on each computer and set-up some software to synchronize your zettel.
Since zettel are stored as ordinary files, this task could be done in various ways.

=== Read-only
You can start the Zettelstore in an read-only mode.
Nobody, not even you as the owner of the Zettelstore, can change something via its interfaces[^However, as an owner, you have access to the files that store the zettel. If you modify the files, these changes will be reflected via its interfaces.].

You enable read-only mode through the key ''readonly'' in the [[start-up configuration zettel|00001004010000]] or with the ''-r'' option of the ``zettelstore run`` sub-command.

=== Authentication
The Zettelstore can be configured that a user must authenticate itself to gain access to the content.

* [[How to enable authentication|00001010040100]]
* [[How to add a new user|00001010040200]]
* [[How users are authenticated|00001010040400]] (some technical background)

Otherwise, an eavesdropper could fetch sensible data, such as passwords or precious content that is not for the public.

The Zettelstore itself does not encrypt messages.
But you can put a server in front of it, which is able to handle encryption.
Most generic web server software do allow this.

To enforce encryption, [[authentication sessions|00001010040700]] are marked as secure by default.
If you still want to access the Zettelstore remotely without encryption, you must change the start-up configuration.
Otherwise, authentication will not work.

* [[Use a server for encryption|00001010090100]]

id: 00001010040100
title: Enable authentication

tags: #authentication #configuration #manual #security #zettelstore
syntax: zmk
role: manual

To enable authentication, you must create a zettel that stores [[authentication data|00001010040200]] for the owner.
Then you must reference this zettel within the [[start-up configuration|00001004010000#owner]] under the key ''owner''.
Once the start-up configuration contains a valid [[zettel identifier|00001006050000]] under that key, authentication is enabled.

id: 00001010040700
title: Access token

tags: #authentication #configuration #manual #security #zettelstore
syntax: zmk
role: manual

If an user is authenticated, an ""access token"" is created that must be sent with every request to prove the identity of the caller.
Otherwise the user will not be recognized by Zettelstore.

If the user was authenticated via the web interface, the access token is stored in a [[""session cookie""|https://en.wikipedia.org/wiki/HTTP_cookie#Session_cookie]].
When the web browser is closed, theses cookies are not saved.
If you want web browser to store the cookie as long as lifetime of that token, the owner must set ''persistent-cookie'' of the [[start-up configuration|00001004010000]] to ''true''.

If the web browser remains inactive for a period, the user will be automatically logged off, because each access token has a limited lifetime.
The maximum length of this period is specified by the ''token-lifetime-html'' value of the start-up configuration.
Every time a web page is displayed, a fresh token is created and stored inside the cookie.

If the user was authenticated via the API, the access token will be returned as the content of the response.
Typically, the lifetime of this token is more short term, e.g. 10 minutes.
It is specified by the ''token-timeout-api'' value of the start-up configuration.
If you need more time, you can either [[re-authenticate|00001012050200]] the user or use an API call to [[renew the access token|00001012050400]].

If you remotely access your Zettelstore via HTTP (not via HTTPS, which allows encrypted communication), your must set the ''insecure-cookie'' value of the start-up configuration to ''true''.
In most cases, such a scenario is not recommended, because user name and password will be transferred as plain text.
You could make use of such scenario if you know all parties that access the local network where you access the Zettelstore.

id: 00001010070200
title: Visibility rules for zettel
role: manual
tags: #authorization #configuration #manual #security #zettelstore
syntax: zmk


For every zettel you can specify under which condition the zettel is visible to others.
This is controlled with the metadata key [[''visibility''|00001006020000#visibility]].
The following values are supported:

; [!public]""public""
: The zettel is visible to everybody, even if the user is not authenticated.
; [!login]""login""
: Only an authenticated user can access the zettel.

  This is the default value for [[''default-visibility''|00001004020000#default-visibility]].
; [!owner]""owner""
: Only the owner of the Zettelstore can access the zettel.

  This is for zettel with sensitive content, e.g. the [[configuration zettel|00001004020000]] or the various zettel that contains the templates for rendering zettel in HTML.
; [!expert]""expert""
: Only the owner of the Zettelstore can access the zettel, if runtime configuration [[''expert-mode''|00001004020000#expert-mode]] is set to a boolean true value.

  This is for zettel with sensitive content that might irritate the owner.
  Computed zettel with internal runtime information are examples for such a zettel.
; [!simple-expert]""simple-expert""
: The owner of the Zettelstore can access the zettel, if expert mode is enabled, or if authentication is disabled and the Zettelstore is started without any command.

  The reason for this is to show all computed zettel to an user that started the Zettestore in simple mode.
  Many computed zettel should be given in error reporting and a new user might not be able to enable expert mode.

When you install a Zettelstore, only two zettel have visibility ""public"".
The first zettel is the zettel that contains CSS for displaying the web interface.
This is to ensure that the web interface looks nice even for not authenticated users.
The other zettel is the zettel containing the [[version|00000000000001]] of the Zettelstore.



Please note: if authentication is not enabled, every user has the same rights as the owner of a Zettelstore.
This is also true, if the Zettelstore runs additionally in [[read-only mode|00001004010000#read-only-mode]].
In this case, the [[runtime configuration zettel|00001004020000]] is shown (its visibility is ""owner"").
The [[start-up configuration|00001004010000]] is not shown, because the associated computed zettel with identifier ''00000000000098'' is stored with the visibility ""expert"".
If you want to show such a zettel, you must set ''expert-mode'' to true.

id: 00001010090100
title: External server to encrypt message transport
role: manual
tags: #configuration #encryption #manual #security #zettelstore
syntax: zmk


Since Zettelstore does not encrypt the messages it exchanges with its clients, you may need some additional software to enable encryption.

=== Public-key encryption
To enable encryption, you probably use some kind of encryption keys.
In most cases, you need to deploy a ""public-key encryption"" process, where your side publish a public encryption key that only works with a corresponding private decryption key.
Technically, this is not trivial.

If you want to add some additional content on the server, you could change the configuration as follows:
```
zettelstore.de {
  file_server * {
    root /var/www/html
  }
  route /manual/* {
    uri strip_prefix /manual
    reverse_proxy localhost:23123
  }
}
```
This will forwards requests with the prefix ""/manual"" to the running Zettelstore.
All other requests will be handled by Caddy itself.

In this case you must specify the start-up configuration key ''url-prefix'' with the value ""/manual"".
This is to allow the Zettelstore to give you the correct URLs with the given prefix.

id: 00001012050200
title: API: Authenticate a client
role: manual
tags: #api #manual #zettelstore
syntax: zmk

Authentication for future API calls is done by sending a [[user identification|00001010040200]] and a password to the Zettelstore to obtain an [[access token|00001010040700]].
This token has to be used for other API calls.
It is valid for a relatively short amount of time, as configured with the key ''token-timeout-api'' of the [[start-up configuration|00001004010000]] (typically 10 minutes).

The simplest way is to send user identification (''IDENT'') and password (''PASSWORD'') via [[HTTP Basic Authentication|https://tools.ietf.org/html/rfc7617]] and send them to the [[endpoint|00001012920000]] ''/a'' with a POST request:
```sh
# curl -X POST -u IDENT:PASSWORD http://127.0.0.1:23123/a
{"access_token":"eyJhbGciOiJIUzUxMiJ9.eyJfdGsiOjEsImV4cCI6MTYwMTczMTI3NSwiaWF0IjoxNjAxNzMwNjc1LCJzdWIiOiJhYmMiLCJ6aWQiOiIyMDIwMTAwMzE1MDEwMCJ9.ekhXkvn146P2bMKFQcU-bNlvgbeO6sS39hs6U5EKfjIqnSInkuHYjYAIfUqf_clYRfr6YBlX5izii8XfxV8jhg","token_type":"Bearer","expires_in":600}
```

id: 00001012051800
title: API: Shape the list of zettel metadata with filter options
role: manual
tags: #api #manual #zettelstore
syntax: zmk


In most cases, it is not essential to list //all// zettel.
Typically, you are interested only in a subset of the zettel maintained by your Zettelstore.
This is done by adding some query parameters to the general ''GET /z'' request.

=== Filter
Every query parameter that does //not// begin with the low line character (""_"", ''U+005F'') is treated as the name of a [[metadata|00001006010000]] key.

{"list":[{"id":"00001012921000","url":"/z/00001012921000","meta":{"title":"API: JSON structure of an access token","tags":"#api #manual #reference #zettelstore","syntax":"zmk","role":"manual"}},{"id":"00001012920500","url":"/z/00001012920500","meta":{"title":"Formats available by the API","tags":"#api #manual #reference #zettelstore","syntax":"zmk","role":"manual"}},{"id":"00001012920000","url":"/z/00001012920000","meta":{"title":"Endpoints used by the API","tags":"#api #manual #reference #zettelstore","syntax":"zmk","role":"manual"}}, ...
```

However, if you want all zettel that does //not// match a given value, you must prefix the value with the exclamation mark character (""!"", ''U+0021'').
For example, if you want to retrieve all zettel that do not contain the string ""API"" in their title, your request will be:
```sh
# curl 'http://127.0.0.1:23123/z?title=!API'
{"list":[{"id":"00010000000000","url":"/z/00010000000000","meta":{"back":"00001003000000 00001005090000","backward":"00001003000000 00001005090000","copyright":"(c) 2020-2021 by Detlef Stern <ds@zettelstore.de>","forward":"00000000000001 00000000000003 00000000000096 00000000000098 00000000000100","lang":"en","license":"EUPL-1.2-or-later","role":"zettel","syntax":"zmk","title":"Home"}},{"id":"00001014000000","url":"/z/00001014000000","meta":{"back":"00001000000000 00001004020000 00001012920510","backward":"00001000000000 00001004020000 00001012000000 00001012920510","copyright":"(c) 2020-2021 by Detlef Stern <ds@zettelstore.de>","forward":"00001012000000","lang":"en","license":"EUPL-1.2-or-later","published":"00001014000000","role":"manual","syntax":"zmk","tags":"#manual #webui #zettelstore","title":"Web user interface"}},
...
```
The empty query parameter values matches all zettel that contain the given metadata key.
Similar, if you specify just the exclamation mark character as a query parameter value, only those zettel match that does //not// contain the given metadata key.
For example ``curl 'http://localhost:23123/z?back=!&backward='`` returns all zettel that are reachable via other zettel, but also references these zettel.

=== Output only specific parts of a zettel

{"list":[{"id":"00001012050200","url":"/z/00001012050200"},{"id":"00001012050400","url":"/z/00001012050400"}]}
```

=== General filter
The query parameter ""''_s''"" allows to provide a string for a full-text search of all zettel.
The search string will be normalized according to Unicode NKFD, ignoring everything except letters and numbers.
























You are allowed to specify this query parameter more than once.
All results will be intersected, i.e. a zettel will be included into the list if both of the provided values match.

This parameter loosely resembles the search box of the web user interface.

id: 00001012920000
title: Endpoints used by the API
role: manual
tags: #api #manual #reference #zettelstore
syntax: zmk


All API endpoints conform to the pattern ''[PREFIX]/LETTER[/ZETTEL-ID]'', where:
; ''PREFIX''
: is an optional URL prefix, configured via the ''url-prefix'' [[start-up configuration|00001004010000]],
; ''LETTER''
: is a single letter that specifies the ressource type,
; ''ZETTEL-ID''
: is an optional 14 digits string that uniquely [[identify a zettel|00001006050000]].

The following letters are currently in use:

type Zid uint64

// Some important ZettelIDs.
// Note: if you change some values, ensure that you also change them in the
//       constant place. They are mentioned there literally, because these
//       constants are not available there.
const (
	Invalid          = Zid(0) // Invalid is a Zid that will never be valid











	ConfigurationZid = Zid(100)

	// WebUI HTML templates are in the range 10000..19999
	BaseTemplateZid    = Zid(10100)
	LoginTemplateZid   = Zid(10200)
	ListTemplateZid    = Zid(10300)
	ZettelTemplateZid  = Zid(10401)
	InfoTemplateZid    = Zid(10402)
	FormTemplateZid    = Zid(10403)
	RenameTemplateZid  = Zid(10404)
	DeleteTemplateZid  = Zid(10405)
	ContextTemplateZid = Zid(10406)
	RolesTemplateZid   = Zid(10500)
	TagsTemplateZid    = Zid(10600)
	ErrorTemplateZid   = Zid(10700)

	// WebUI CSS pages are in the range 20000..29999
	BaseCSSZid = Zid(20001)

	// WebUI JS pages are in the range 30000..39999




	// Range 90000...99999 is reserved for zettel templates
	TOCNewTemplateZid    = Zid(90000)
	TemplateNewZettelZid = Zid(90001)
	TemplateNewUserZid   = Zid(90002)

	DefaultHomeZid = Zid(10000000000)
)

const maxZid = 99999999999999














// Parse interprets a string as a zettel identification and
// returns its integer value.
func Parse(s string) (Zid, error) {
	if len(s) != 14 {
		return Invalid, strconv.ErrSyntax
	}
	res, err := strconv.ParseUint(s, 10, 47)
	if err != nil {
		return Invalid, err
	}
	if res == 0 {
		return Invalid, strconv.ErrRange
	}
	return Zid(res), nil
}

const digits = "0123456789"

// String converts the zettel identification to a string of 14 digits.
// Only defined for valid ids.

				"i=%d: zid=%v does not format to %q, but to %q", i, sid, zid, s)
		}
	}

	invalidIDs := []string{
		"", "0", "a",
		"00000000000000",

		"000000000000000",
		"99999999999999a",
		"20200310T195100",
	}

	for i, zid := range invalidIDs {
		if _, err := id.Parse(zid); err == nil {
			t.Errorf("i=%d: zid=%q is valid, but should not be", i, zid)
		}
	}
}

// NewSet returns a new set of identifier with the given initial values.
func NewSet(zids ...Zid) Set {
	l := len(zids)
	if l < 8 {
		l = 8
	}
	result := make(Set, l)
	for _, zid := range zids {
		result[zid] = true
	}
	return result
}

// NewSetCap returns a new set of identifier with the given capacity and initial values.
func NewSetCap(c int, zids ...Zid) Set {
	l := len(zids)
	if c < l {
		c = l
	}
	if c < 8 {
		c = 8
	}
	result := make(Set, c)






	for _, zid := range zids {
		result[zid] = true
	}
	return result
}

// Sorted returns the set as a sorted slice of zettel identifier.
func (s Set) Sorted() Slice {
	if l := len(s); l > 0 {
		result := make(Slice, 0, l)
		for zid := range s {
			result = append(result, zid)

		otherInSet, otherOk := other[zid]
		if !otherInSet || !otherOk {
			delete(s, zid)
		}
	}
	return s
}

		}
		got = id.NewSet(sl2...).Intersect(id.NewSet(sl1...)).Sorted()
		if !got.Equal(tc.exp) {
			t.Errorf("%d: %v.Intersect(%v) should be %v, but got %v", i, sl2, sl1, tc.exp, got)
		}
	}
}

// Important values for some keys.
const (
	ValueRoleConfiguration = "configuration"
	ValueRoleUser          = "user"
	ValueRoleZettel        = "zettel"
	ValueSyntaxNone        = "none"


	ValueSyntaxZmk         = "zmk"
	ValueTrue              = "true"
	ValueFalse             = "false"

	ValueUserRoleReader    = "reader"
	ValueUserRoleWriter    = "writer"
	ValueUserRoleOwner     = "owner"
	ValueVisibilityExpert  = "expert"
	ValueVisibilityOwner   = "owner"
	ValueVisibilityLogin   = "login"
	ValueVisibilityPublic  = "public"
	ValueVisibilitySimple  = "simple-expert"
)

// Meta contains all meta-data of a zettel.
type Meta struct {
	Zid     id.Zid
	pairs   map[string]string
	YamlSep bool

//-----------------------------------------------------------------------------
// Copyright (c) 2020 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Supported values for visibility.
const (
	_ Visibility = iota
	VisibilityUnknown
	VisibilityPublic
	VisibilityLogin
	VisibilityOwner
	VisibilitySimple
	VisibilityExpert
)

var visMap = map[string]Visibility{
	ValueVisibilityPublic: VisibilityPublic,
	ValueVisibilityLogin:  VisibilityLogin,
	ValueVisibilityOwner:  VisibilityOwner,
	ValueVisibilitySimple: VisibilitySimple,
	ValueVisibilityExpert: VisibilityExpert,
}

// GetVisibility returns the visibility value of the given string
func GetVisibility(val string) Visibility {
	if vis, ok := visMap[val]; ok {
		return vis

				v.b.WriteByte('\n')
			}
			v.b.WriteString("-->\n")
		}

	case ast.VerbatimHTML:
		for _, line := range vn.Lines {

			v.b.WriteStrings(line, "\n")

		}
	default:
		panic(fmt.Sprintf("Unknown verbatim code %v", vn.Code))
	}
}


















var specialSpanAttr = map[string]bool{
	"example":   true,
	"note":      true,
	"tip":       true,
	"important": true,
	"caution":   true,

import (
	"fmt"
	"strconv"
	"strings"

	"zettelstore.de/z/ast"

)

// VisitText writes text content.
func (v *visitor) VisitText(tn *ast.TextNode) {
	v.writeHTMLEscaped(tn.Text)
}

	v.b.WriteByte('>')
	v.acceptInlineSlice(ins)
	v.b.WriteString("</span>")

}

var langQuotes = map[string][2]string{
	"en": {"&ldquo;", "&rdquo;"},
	"de": {"&bdquo;", "&ldquo;"},
	"fr": {"&laquo;&nbsp;", "&nbsp;&raquo;"},
}

func getQuotes(lang string) (string, string) {
	langFields := strings.FieldsFunc(lang, func(r rune) bool { return r == '-' || r == '_' })
	for len(langFields) > 0 {
		langSup := strings.Join(langFields, "-")
		quotes, ok := langQuotes[langSup]

	case ast.LiteralOutput:
		v.writeLiteral("<samp", "</samp>", ln.Attrs, ln.Text)
	case ast.LiteralComment:
		v.b.WriteString("<!-- ")
		v.writeHTMLEscaped(ln.Text) // writeCommentEscaped
		v.b.WriteString(" -->")
	case ast.LiteralHTML:

		v.b.WriteString(ln.Text)

	default:
		panic(fmt.Sprintf("Unknown literal code %v", ln.Code))
	}
}

func (v *visitor) writeLiteral(codeS, codeE string, attrs *ast.Attributes, text string) {
	oldVisible := v.visibleSpace

		if i < len(tn.Text)-1 {
			s := tn.Text[i : i+2]
			if _, ok := escapeSeqs[s]; ok {
				v.b.WriteString(tn.Text[last:i])
				for j := 0; j < len(s); j++ {
					v.b.WriteBytes('\\', s[j])
				}

				last = i + 1
				continue
			}
		}
	}
	v.b.WriteString(tn.Text[last:])
}

module zettelstore.de/z

go 1.16

require (
	github.com/fsnotify/fsnotify v1.4.9
	github.com/pascaldekloe/jwt v1.10.0
	github.com/yuin/goldmark v1.3.5
	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
	golang.org/x/term v0.0.0-20201117132131-f5c789dd3221
	golang.org/x/text v0.3.6
)

github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/pascaldekloe/jwt v1.10.0 h1:ktcIUV4TPvh404R5dIBEnPCsSwj0sqi3/0+XafE5gJs=
github.com/pascaldekloe/jwt v1.10.0/go.mod h1:TKhllgThT7TOP5rGr2zMLKEDZRAgJfBbtKyVeRsNB9A=
github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package index allows to search for metadata and content.
package index

import (
	"context"

	"zettelstore.de/z/domain/meta"
)

// Remover is used to remove some metadata before they are stored in a place.
type Remover interface {
	// Remove removes computed properties from the given metadata.
	// It is called by the manager place before meta data is updated.
	Remove(ctx context.Context, m *meta.Meta)
}

// MetaFilter is used by places to filter and set computed metadata value.
type MetaFilter interface {
	Enricher
	Remover
}

type metaFilter struct {
	index      Indexer
	properties map[string]bool // Set of property key names
}

// NewMetaFilter creates a new meta filter.
func NewMetaFilter(idx Indexer) MetaFilter {
	properties := make(map[string]bool)
	for _, kd := range meta.GetSortedKeyDescriptions() {
		if kd.IsProperty() {
			properties[kd.Name] = true
		}
	}
	return &metaFilter{
		index:      idx,
		properties: properties,
	}
}

func (mf *metaFilter) Enrich(ctx context.Context, m *meta.Meta) {
	computePublished(m)
	mf.index.Enrich(ctx, m)
}

func computePublished(m *meta.Meta) {
	if _, ok := m.Get(meta.KeyPublished); ok {
		return
	}
	if modified, ok := m.Get(meta.KeyModified); ok {
		if _, ok = meta.TimeValue(modified); ok {
			m.Set(meta.KeyPublished, modified)
			return
		}
	}
	zid := m.Zid.String()
	if _, ok := meta.TimeValue(zid); ok {
		m.Set(meta.KeyPublished, zid)
		return
	}

	// Neither the zettel was modified nor the zettel identifer contains a valid
	// timestamp. In this case do not set the "published" property.
}

func (mf *metaFilter) Remove(ctx context.Context, m *meta.Meta) {
	for _, p := range m.PairsRest(true) {
		if mf.properties[p.Key] {
			m.Delete(p.Key)
		}
	}
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package index allows to search for metadata and content.
package index

import (
	"context"
	"io"
	"time"

	"zettelstore.de/z/domain"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/place/change"
)

// Enricher is used to update metadata by adding new properties.
type Enricher interface {
	// Enrich computes additional properties and updates the given metadata.
	// It is typically called by zettel reading methods.
	Enrich(ctx context.Context, m *meta.Meta)
}

// Selector is used to select zettel identifier based on selection criteria.
type Selector interface {
	// Select all zettel that contains the given exact word.
	// The word must be normalized through Unicode NKFD.
	Select(word string) id.Set

	// Select all zettel that have a word with the given prefix.
	// The prefix must be normalized through Unicode NKFD.
	SelectPrefix(prefix string) id.Set

	// Select all zettel that contains the given string.
	// The string must be normalized through Unicode NKFD.
	SelectContains(s string) id.Set
}

// NoEnrichContext will signal an enricher that nothing has to be done.
// This is useful for an Indexer, but also for some place.Place calls, when
// just the plain metadata is needed.
func NoEnrichContext(ctx context.Context) context.Context {
	return context.WithValue(ctx, ctxNoEnrichKey, &ctxNoEnrichKey)
}

type ctxNoEnrichType struct{}

var ctxNoEnrichKey ctxNoEnrichType

// DoNotEnrich determines if the context is marked to not enrich metadata.
func DoNotEnrich(ctx context.Context) bool {
	_, ok := ctx.Value(ctxNoEnrichKey).(*ctxNoEnrichType)
	return ok
}

// Port contains all the used functions to access zettel to be indexed.
type Port interface {
	RegisterObserver(change.Func)
	FetchZids(context.Context) (id.Set, error)
	GetMeta(context.Context, id.Zid) (*meta.Meta, error)
	GetZettel(context.Context, id.Zid) (domain.Zettel, error)
}

// Indexer contains all the functions of an index.
type Indexer interface {
	Enricher
	Selector

	// Start the index. It will read all zettel and store index data for later retrieval.
	Start(Port)

	// Stop the index. No zettel are read any more, but the current index data
	// can stil be retrieved.
	Stop()

	// ReadStats populates st with indexer statistics.
	ReadStats(st *IndexerStats)
}

// IndexerStats records statistics about the indexer.
type IndexerStats struct {
	// LastReload stores the timestamp when a full re-index was done.
	LastReload time.Time

	// IndexesSinceReload counts indexing a zettel since the full re-index.
	IndexesSinceReload uint64

	// DurLastIndex is the duration of the last index run. This could be a
	// full re-index or a re-index of a single zettel.
	DurLastIndex time.Duration

	// Store records statistics about the underlying index store.
	Store StoreStats
}

// Store all relevant zettel data. There may be multiple implementations, i.e.
// memory-based, file-based, based on SQLite, ...
type Store interface {
	Enricher
	Selector

	// UpdateReferences for a specific zettel.
	// Returns set of zettel identifier that must also be checked for changes.
	UpdateReferences(context.Context, *ZettelIndex) id.Set

	// DeleteZettel removes index data for given zettel.
	// Returns set of zettel identifier that must also be checked for changes.
	DeleteZettel(context.Context, id.Zid) id.Set

	// ReadStats populates st with store statistics.
	ReadStats(st *StoreStats)

	// Write the content to a Writer.
	Write(io.Writer)
}

// StoreStats records statistics about the store.
type StoreStats struct {
	// Zettel is the number of zettel managed by the indexer.
	Zettel int

	// Updates count the number of metadata updates.
	Updates uint64

	// Words count the different words stored in the store.
	Words uint64
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package indexer allows to search for metadata and content.
package indexer

import (
	"sync"

	"zettelstore.de/z/domain/id"
)

type arAction int

const (
	arNothing arAction = iota
	arReload
	arUpdate
	arDelete
)

type anteroom struct {
	next    *anteroom
	waiting map[id.Zid]arAction
	curLoad int
	reload  bool
}

type anterooms struct {
	mx      sync.Mutex
	first   *anteroom
	last    *anteroom
	maxLoad int
}

func newAnterooms(maxLoad int) *anterooms {
	return &anterooms{maxLoad: maxLoad}
}

func (ar *anterooms) Enqueue(zid id.Zid, action arAction) {
	if !zid.IsValid() || action == arNothing || action == arReload {
		return
	}
	ar.mx.Lock()
	defer ar.mx.Unlock()
	if ar.first == nil {
		ar.first = ar.makeAnteroom(zid, action)
		ar.last = ar.first
		return
	}
	for room := ar.first; room != nil; room = room.next {
		if room.reload {
			continue // Do not place zettel in reload room
		}
		a, ok := room.waiting[zid]
		if !ok {
			continue
		}
		switch action {
		case a:
			return
		case arUpdate:
			room.waiting[zid] = action
		case arDelete:
			room.waiting[zid] = action
		}
		return
	}
	if room := ar.last; !room.reload && (ar.maxLoad == 0 || room.curLoad < ar.maxLoad) {
		room.waiting[zid] = action
		room.curLoad++
		return
	}
	room := ar.makeAnteroom(zid, action)
	ar.last.next = room
	ar.last = room
}

func (ar *anterooms) makeAnteroom(zid id.Zid, action arAction) *anteroom {
	c := ar.maxLoad
	if c == 0 {
		c = 100
	}
	waiting := make(map[id.Zid]arAction, c)
	waiting[zid] = action
	return &anteroom{next: nil, waiting: waiting, curLoad: 1, reload: false}
}

func (ar *anterooms) Reset() {
	ar.mx.Lock()
	defer ar.mx.Unlock()
	ar.first = ar.makeAnteroom(id.Invalid, arReload)
	ar.last = ar.first
}

func (ar *anterooms) Reload(delZids id.Slice, newZids id.Set) {
	ar.mx.Lock()
	defer ar.mx.Unlock()
	delWaiting := createWaitingSlice(delZids, arDelete)
	newWaiting := createWaitingSet(newZids, arUpdate)
	ar.deleteReloadedRooms()

	if ds := len(delWaiting); ds > 0 {
		if ns := len(newWaiting); ns > 0 {
			roomNew := &anteroom{next: ar.first, waiting: newWaiting, curLoad: ns, reload: true}
			ar.first = &anteroom{next: roomNew, waiting: delWaiting, curLoad: ds, reload: true}
			if roomNew.next == nil {
				ar.last = roomNew
			}
			return
		}

		ar.first = &anteroom{next: ar.first, waiting: delWaiting, curLoad: ds}
		if ar.first.next == nil {
			ar.last = ar.first
		}
		return
	}

	if ns := len(newWaiting); ns > 0 {
		ar.first = &anteroom{next: ar.first, waiting: newWaiting, curLoad: ns}
		if ar.first.next == nil {
			ar.last = ar.first
		}
		return
	}

	ar.first = nil
	ar.last = nil
}

func createWaitingSlice(zids id.Slice, action arAction) map[id.Zid]arAction {
	waitingSet := make(map[id.Zid]arAction, len(zids))
	for _, zid := range zids {
		if zid.IsValid() {
			waitingSet[zid] = action
		}
	}
	return waitingSet
}

func createWaitingSet(zids id.Set, action arAction) map[id.Zid]arAction {
	waitingSet := make(map[id.Zid]arAction, len(zids))
	for zid := range zids {
		if zid.IsValid() {
			waitingSet[zid] = action
		}
	}
	return waitingSet
}

func (ar *anterooms) deleteReloadedRooms() {
	room := ar.first
	for room != nil && room.reload {
		room = room.next
	}
	ar.first = room
	if room == nil {
		ar.last = nil
	}
}

func (ar *anterooms) Dequeue() (arAction, id.Zid) {
	ar.mx.Lock()
	defer ar.mx.Unlock()
	if ar.first == nil {
		return arNothing, id.Invalid
	}
	for zid, action := range ar.first.waiting {
		delete(ar.first.waiting, zid)
		if len(ar.first.waiting) == 0 {
			ar.first = ar.first.next
			if ar.first == nil {
				ar.last = nil
			}
		}
		return action, zid
	}
	return arNothing, id.Invalid
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package indexer allows to search for metadata and content.
package indexer

import (
	"testing"

	"zettelstore.de/z/domain/id"
)

func TestSimple(t *testing.T) {
	ar := newAnterooms(2)
	ar.Enqueue(id.Zid(1), arUpdate)
	action, zid := ar.Dequeue()
	if zid != id.Zid(1) || action != arUpdate {
		t.Errorf("Expected 1/arUpdate, but got %v/%v", zid, action)
	}
	action, zid = ar.Dequeue()
	if zid != id.Invalid && action != arDelete {
		t.Errorf("Expected invalid Zid, but got %v", zid)
	}
	ar.Enqueue(id.Zid(1), arUpdate)
	ar.Enqueue(id.Zid(2), arUpdate)
	if ar.first != ar.last {
		t.Errorf("Expected one room, but got more")
	}
	ar.Enqueue(id.Zid(3), arUpdate)
	if ar.first == ar.last {
		t.Errorf("Expected more than one room, but got only one")
	}

	count := 0
	for ; count < 1000; count++ {
		action, _ := ar.Dequeue()
		if action == arNothing {
			break
		}
	}
	if count != 3 {
		t.Errorf("Expected 3 dequeues, but got %v", count)
	}
}

func TestReset(t *testing.T) {
	ar := newAnterooms(1)
	ar.Enqueue(id.Zid(1), arUpdate)
	ar.Reset()
	action, zid := ar.Dequeue()
	if action != arReload || zid != id.Invalid {
		t.Errorf("Expected reload & invalid Zid, but got %v/%v", action, zid)
	}
	ar.Reload(id.Slice{2}, id.NewSet(3, 4))
	ar.Enqueue(id.Zid(5), arUpdate)
	ar.Enqueue(id.Zid(5), arDelete)
	ar.Enqueue(id.Zid(5), arDelete)
	ar.Enqueue(id.Zid(5), arUpdate)
	if ar.first == ar.last || ar.first.next == ar.last || ar.first.next.next != ar.last {
		t.Errorf("Expected 3 rooms")
	}
	action, zid = ar.Dequeue()
	if zid != id.Zid(2) || action != arDelete {
		t.Errorf("Expected 2/arDelete, but got %v/%v", zid, action)
	}
	action, zid1 := ar.Dequeue()
	if action != arUpdate {
		t.Errorf("Expected arUpdate, but got %v", action)
	}
	action, zid2 := ar.Dequeue()
	if action != arUpdate {
		t.Errorf("Expected arUpdate, but got %v", action)
	}
	if !(zid1 == id.Zid(3) && zid2 == id.Zid(4) || zid1 == id.Zid(4) && zid2 == id.Zid(3)) {
		t.Errorf("Zids must be 3 or 4, but got %v/%v", zid1, zid2)
	}
	action, zid = ar.Dequeue()
	if zid != id.Zid(5) || action != arUpdate {
		t.Errorf("Expected 5/arUpdate, but got %v/%v", zid, action)
	}
	action, zid = ar.Dequeue()
	if action != arNothing || zid != id.Invalid {
		t.Errorf("Expected nothing & invalid Zid, but got %v/%v", action, zid)
	}

	ar = newAnterooms(1)
	ar.Reload(nil, id.NewSet(id.Zid(6)))
	action, zid = ar.Dequeue()
	if zid != id.Zid(6) || action != arUpdate {
		t.Errorf("Expected 6/arUpdate, but got %v/%v", zid, action)
	}
	action, zid = ar.Dequeue()
	if action != arNothing || zid != id.Invalid {
		t.Errorf("Expected nothing & invalid Zid, but got %v/%v", action, zid)
	}

	ar = newAnterooms(1)
	ar.Reload(id.Slice{7}, nil)
	action, zid = ar.Dequeue()
	if zid != id.Zid(7) || action != arDelete {
		t.Errorf("Expected 7/arDelete, but got %v/%v", zid, action)
	}
	action, zid = ar.Dequeue()
	if action != arNothing || zid != id.Invalid {
		t.Errorf("Expected nothing & invalid Zid, but got %v/%v", action, zid)
	}

	ar = newAnterooms(1)
	ar.Enqueue(id.Zid(8), arUpdate)
	ar.Reload(nil, nil)
	action, zid = ar.Dequeue()
	if action != arNothing || zid != id.Invalid {
		t.Errorf("Expected nothing & invalid Zid, but got %v/%v", action, zid)
	}
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package indexer allows to search for metadata and content.
package indexer

import (
	"zettelstore.de/z/ast"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/index"
	"zettelstore.de/z/strfun"
)

func collectZettelIndexData(zn *ast.ZettelNode, refs id.Set, words index.WordSet) {
	ixv := ixVisitor{refs: refs, words: words}
	ast.NewTopDownTraverser(&ixv).VisitBlockSlice(zn.Ast)
}

func collectInlineIndexData(ins ast.InlineSlice, refs id.Set, words index.WordSet) {
	ixv := ixVisitor{refs: refs, words: words}
	ast.NewTopDownTraverser(&ixv).VisitInlineSlice(ins)
}

type ixVisitor struct {
	refs  id.Set
	words index.WordSet
}

// VisitVerbatim collects the verbatim text in the word set.
func (lv *ixVisitor) VisitVerbatim(vn *ast.VerbatimNode) {
	for _, line := range vn.Lines {
		lv.addText(line)
	}
}

// VisitRegion does nothing.
func (lv *ixVisitor) VisitRegion(rn *ast.RegionNode) {}

// VisitHeading does nothing.
func (lv *ixVisitor) VisitHeading(hn *ast.HeadingNode) {}

// VisitHRule does nothing.
func (lv *ixVisitor) VisitHRule(hn *ast.HRuleNode) {}

// VisitList does nothing.
func (lv *ixVisitor) VisitNestedList(ln *ast.NestedListNode) {}

// VisitDescriptionList does nothing.
func (lv *ixVisitor) VisitDescriptionList(dn *ast.DescriptionListNode) {}

// VisitPara does nothing.
func (lv *ixVisitor) VisitPara(pn *ast.ParaNode) {}

// VisitTable does nothing.
func (lv *ixVisitor) VisitTable(tn *ast.TableNode) {}

// VisitBLOB does nothing.
func (lv *ixVisitor) VisitBLOB(bn *ast.BLOBNode) {}

// VisitText collects the text in the word set.
func (lv *ixVisitor) VisitText(tn *ast.TextNode) {
	lv.addText(tn.Text)
}

// VisitTag collects the tag name in the word set.
func (lv *ixVisitor) VisitTag(tn *ast.TagNode) {
	lv.addText(tn.Tag)
}

// VisitSpace does nothing.
func (lv *ixVisitor) VisitSpace(sn *ast.SpaceNode) {}

// VisitBreak does nothing.
func (lv *ixVisitor) VisitBreak(bn *ast.BreakNode) {}

// VisitLink collects the given link as a reference.
func (lv *ixVisitor) VisitLink(ln *ast.LinkNode) {
	ref := ln.Ref
	if ref == nil || !ref.IsZettel() {
		return
	}
	if zid, err := id.Parse(ref.URL.Path); err == nil {
		lv.refs[zid] = true
	}
}

// VisitImage collects the image links as a reference.
func (lv *ixVisitor) VisitImage(in *ast.ImageNode) {
	ref := in.Ref
	if ref == nil || !ref.IsZettel() {
		return
	}
	if zid, err := id.Parse(ref.URL.Path); err == nil {
		lv.refs[zid] = true
	}
}

// VisitCite does nothing.
func (lv *ixVisitor) VisitCite(cn *ast.CiteNode) {}

// VisitFootnote does nothing.
func (lv *ixVisitor) VisitFootnote(fn *ast.FootnoteNode) {}

// VisitMark does nothing.
func (lv *ixVisitor) VisitMark(mn *ast.MarkNode) {}

// VisitFormat does nothing.
func (lv *ixVisitor) VisitFormat(fn *ast.FormatNode) {}

// VisitLiteral collects the literal words in the word set.
func (lv *ixVisitor) VisitLiteral(ln *ast.LiteralNode) {
	lv.addText(ln.Text)
}

func (lv *ixVisitor) addText(s string) {
	for _, word := range strfun.NormalizeWords(s) {
		lv.words[word] = lv.words[word] + 1
	}
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package indexer allows to search for metadata and content.
package indexer

import (
	"context"
	"log"
	"runtime/debug"
	"sync"
	"time"

	"zettelstore.de/z/domain"
	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/index"
	"zettelstore.de/z/index/memstore"
	"zettelstore.de/z/parser"
	"zettelstore.de/z/place/change"
	"zettelstore.de/z/strfun"
)

type indexer struct {
	store   index.Store
	ar      *anterooms
	ready   chan struct{} // Signal a non-empty anteroom to background task
	done    chan struct{} // Stop background task
	observe bool
	started bool

	// Stats data
	mx           sync.RWMutex
	lastReload   time.Time
	sinceReload  uint64
	durLastIndex time.Duration
}

// New creates a new indexer.
func New() index.Indexer {
	return &indexer{
		store: memstore.New(),
		ar:    newAnterooms(10),
		ready: make(chan struct{}, 1),
	}
}

func (idx *indexer) observer(ci change.Info) {
	switch ci.Reason {
	case change.OnReload:
		idx.ar.Reset()
	case change.OnUpdate:
		idx.ar.Enqueue(ci.Zid, arUpdate)
	case change.OnDelete:
		idx.ar.Enqueue(ci.Zid, arDelete)
	default:
		return
	}
	select {
	case idx.ready <- struct{}{}:
	default:
	}
}

func (idx *indexer) Start(p index.Port) {
	if idx.started {
		panic("Index already started")
	}
	idx.done = make(chan struct{})
	if !idx.observe {
		p.RegisterObserver(idx.observer)
		idx.observe = true
	}
	idx.ar.Reset() // Ensure an initial index run
	go idx.indexer(p)
	idx.started = true
}

func (idx *indexer) Stop() {
	if !idx.started {
		panic("Index already stopped")
	}
	close(idx.done)
	idx.started = false
}

// Enrich reads all properties in the index and updates the metadata.
func (idx *indexer) Enrich(ctx context.Context, m *meta.Meta) {
	if index.DoNotEnrich(ctx) {
		// Enrich is called indirectly via indexer or enrichment is not requested
		// because of other reasons -> ignore this call, do not update meta data
		return
	}
	idx.store.Enrich(ctx, m)
}

// Select all zettel that contains the given exact word.
// The word must be normalized through Unicode NKFD.
func (idx *indexer) Select(word string) id.Set {
	return idx.store.Select(word)
}

// Select all zettel that have a word with the given prefix.
// The prefix must be normalized through Unicode NKFD.
func (idx *indexer) SelectPrefix(prefix string) id.Set {
	return idx.store.SelectPrefix(prefix)
}

// Select all zettel that contains the given string.
// The string must be normalized through Unicode NKFD.
func (idx *indexer) SelectContains(s string) id.Set {
	return idx.store.SelectContains(s)
}

// ReadStats populates st with indexer statistics.
func (idx *indexer) ReadStats(st *index.IndexerStats) {
	idx.mx.RLock()
	st.LastReload = idx.lastReload
	st.IndexesSinceReload = idx.sinceReload
	st.DurLastIndex = idx.durLastIndex
	idx.mx.RUnlock()
	idx.store.ReadStats(&st.Store)
}

type indexerPort interface {
	getMetaPort
	FetchZids(ctx context.Context) (id.Set, error)
	GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error)
}

// indexer runs in the background and updates the index data structures.
// This is the main service of the indexer.
func (idx *indexer) indexer(p indexerPort) {
	// Something may panic. Ensure a running indexer.
	defer func() {
		if r := recover(); r != nil {
			log.Println("recovered from:", r)
			debug.PrintStack()
			go idx.indexer(p)
		}
	}()

	timerDuration := 15 * time.Second
	timer := time.NewTimer(timerDuration)
	ctx := index.NoEnrichContext(context.Background())
	for {
		start := time.Now()
		if idx.workService(ctx, p) {
			idx.mx.Lock()
			idx.durLastIndex = time.Since(start)
			idx.mx.Unlock()
		}
		if !idx.sleepService(timer, timerDuration) {
			return
		}
	}
}

func (idx *indexer) workService(ctx context.Context, p indexerPort) bool {
	changed := false
	for {
		switch action, zid := idx.ar.Dequeue(); action {
		case arNothing:
			return changed
		case arReload:
			zids, err := p.FetchZids(ctx)
			if err == nil {
				idx.ar.Reload(nil, zids)
				idx.mx.Lock()
				idx.lastReload = time.Now()
				idx.sinceReload = 0
				idx.mx.Unlock()
			}
		case arUpdate:
			changed = true
			idx.mx.Lock()
			idx.sinceReload++
			idx.mx.Unlock()
			zettel, err := p.GetZettel(ctx, zid)
			if err != nil {
				// TODO: on some errors put the zid into a "try later" set
				continue
			}
			idx.updateZettel(ctx, zettel, p)
		case arDelete:
			changed = true
			idx.mx.Lock()
			idx.sinceReload++
			idx.mx.Unlock()
			idx.deleteZettel(zid)
		}
	}
}

func (idx *indexer) sleepService(timer *time.Timer, timerDuration time.Duration) bool {
	select {
	case _, ok := <-idx.ready:
		if !ok {
			return false
		}
	case _, ok := <-timer.C:
		if !ok {
			return false
		}
		timer.Reset(timerDuration)
	case _, ok := <-idx.done:
		if !ok {
			if !timer.Stop() {
				<-timer.C
			}
			return false
		}
	}
	return true
}

type getMetaPort interface {
	GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error)
}

func (idx *indexer) updateZettel(ctx context.Context, zettel domain.Zettel, p getMetaPort) {
	m := zettel.Meta
	if m.GetBool(meta.KeyNoIndex) {
		// Zettel maybe in index
		toCheck := idx.store.DeleteZettel(ctx, m.Zid)
		idx.checkZettel(toCheck)
		return
	}
	refs := id.NewSet()
	words := make(index.WordSet)
	collectZettelIndexData(parser.ParseZettel(zettel, ""), refs, words)
	zi := index.NewZettelIndex(m.Zid)
	for _, pair := range m.Pairs(false) {
		descr := meta.GetDescription(pair.Key)
		if descr.IsComputed() {
			continue
		}
		switch descr.Type {
		case meta.TypeID:
			updateValue(ctx, descr.Inverse, pair.Value, p, zi)
		case meta.TypeIDSet:
			for _, val := range meta.ListFromValue(pair.Value) {
				updateValue(ctx, descr.Inverse, val, p, zi)
			}
		case meta.TypeZettelmarkup:
			collectInlineIndexData(parser.ParseMetadata(pair.Value), refs, words)
		default:
			for _, word := range strfun.NormalizeWords(pair.Value) {
				words[word] = words[word] + 1
			}
		}
	}
	for ref := range refs {
		if _, err := p.GetMeta(ctx, ref); err == nil {
			zi.AddBackRef(ref)
		} else {
			zi.AddDeadRef(ref)
		}
	}
	zi.SetWords(words)
	toCheck := idx.store.UpdateReferences(ctx, zi)
	idx.checkZettel(toCheck)
}

func updateValue(ctx context.Context, inverse string, value string, p getMetaPort, zi *index.ZettelIndex) {
	zid, err := id.Parse(value)
	if err != nil {
		return
	}
	if _, err := p.GetMeta(ctx, zid); err != nil {
		zi.AddDeadRef(zid)
		return
	}
	if inverse == "" {
		zi.AddBackRef(zid)
		return
	}
	zi.AddMetaRef(inverse, zid)
}

func (idx *indexer) deleteZettel(zid id.Zid) {
	toCheck := idx.store.DeleteZettel(context.Background(), zid)
	idx.checkZettel(toCheck)
}

func (idx *indexer) checkZettel(s id.Set) {
	for zid := range s {
		idx.ar.Enqueue(zid, arUpdate)
	}
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package memstore stored the index in main memory.
package memstore

import (
	"context"
	"fmt"
	"io"
	"strings"
	"sync"

	"zettelstore.de/z/domain/id"
	"zettelstore.de/z/domain/meta"
	"zettelstore.de/z/index"
)

type metaRefs struct {
	forward  id.Slice
	backward id.Slice
}

type zettelIndex struct {
	dead     id.Slice
	forward  id.Slice
	backward id.Slice
	meta     map[string]metaRefs
	words    []string
}

func (zi *zettelIndex) isEmpty() bool {
	if len(zi.forward) > 0 || len(zi.backward) > 0 || len(zi.dead) > 0 || len(zi.words) > 0 {
		return false
	}
	return zi.meta == nil || len(zi.meta) == 0
}

type memStore struct {
	mx    sync.RWMutex
	idx   map[id.Zid]*zettelIndex
	dead  map[id.Zid]id.Slice // map dead refs where they occur
	words map[string]id.Slice

	// Stats
	updates uint64
}

// New returns a new memory-based index store.
func New() index.Store {
	return &memStore{
		idx:   make(map[id.Zid]*zettelIndex),
		dead:  make(map[id.Zid]id.Slice),
		words: make(map[string]id.Slice),
	}
}

func (ms *memStore) Enrich(ctx context.Context, m *meta.Meta) {
	if ms.doEnrich(ctx, m) {
		ms.mx.Lock()
		ms.updates++
		ms.mx.Unlock()
	}
}

func (ms *memStore) doEnrich(ctx context.Context, m *meta.Meta) bool {
	ms.mx.RLock()
	defer ms.mx.RUnlock()
	zi, ok := ms.idx[m.Zid]
	if !ok {
		return false
	}
	var updated bool
	if len(zi.dead) > 0 {
		m.Set(meta.KeyDead, zi.dead.String())
		updated = true
	}
	back := removeOtherMetaRefs(m, zi.backward.Copy())
	if len(zi.backward) > 0 {
		m.Set(meta.KeyBackward, zi.backward.String())
		updated = true
	}
	if len(zi.forward) > 0 {
		m.Set(meta.KeyForward, zi.forward.String())
		back = remRefs(back, zi.forward)
		updated = true
	}
	if len(zi.meta) > 0 {
		for k, refs := range zi.meta {
			if len(refs.backward) > 0 {
				m.Set(k, refs.backward.String())
				back = remRefs(back, refs.backward)
				updated = true
			}
		}
	}
	if len(back) > 0 {
		m.Set(meta.KeyBack, back.String())
		updated = true
	}
	return updated
}

// Select all zettel that contains the given exact word.
// The word must be normalized through Unicode NKFD.
func (ms *memStore) Select(word string) id.Set {
	ms.mx.RLock()
	defer ms.mx.RUnlock()
	if refs, ok := ms.words[word]; ok {
		return id.NewSet(refs...)
	}
	return nil
}

// Select all zettel that have a word with the given prefix.
// The prefix must be normalized through Unicode NKFD.
func (ms *memStore) SelectPrefix(prefix string) id.Set {
	ms.mx.RLock()
	defer ms.mx.RUnlock()
	return ms.selectWithPred(prefix, strings.HasPrefix)
}

// Select all zettel that contains the given string.
// The string must be normalized through Unicode NKFD.
func (ms *memStore) SelectContains(s string) id.Set {
	ms.mx.RLock()
	defer ms.mx.RUnlock()
	return ms.selectWithPred(s, strings.Contains)
}

func (ms *memStore) selectWithPred(s string, pred func(string, string) bool) id.Set {
	result := id.NewSet()
	for word, refs := range ms.words {
		if !pred(word, s) {
			continue
		}
		for _, ref := range refs {
			result[ref] = true
		}
	}
	return result
}

func removeOtherMetaRefs(m *meta.Meta, back id.Slice) id.Slice {
	for _, p := range m.PairsRest(false) {
		switch meta.Type(p.Key) {
		case meta.TypeID:
			if zid, err := id.Parse(p.Value); err == nil {
				back = remRef(back, zid)
			}
		case meta.TypeIDSet:
			for _, val := range meta.ListFromValue(p.Value) {
				if zid, err := id.Parse(val); err == nil {
					back = remRef(back, zid)
				}
			}
		}
	}
	return back
}

func (ms *memStore) UpdateReferences(ctx context.Context, zidx *index.ZettelIndex) id.Set {
	ms.mx.Lock()
	defer ms.mx.Unlock()
	zi, ziExist := ms.idx[zidx.Zid]
	if !ziExist || zi == nil {
		zi = &zettelIndex{}
		ziExist = false
	}

	// Is this zettel an old dead reference mentioned in other zettel?
	var toCheck id.Set
	if refs, ok := ms.dead[zidx.Zid]; ok {
		// These must be checked later again
		toCheck = id.NewSet(refs...)
		delete(ms.dead, zidx.Zid)
	}

	ms.updateDeadReferences(zidx, zi)
	ms.updateForwardBackwardReferences(zidx, zi)
	ms.updateMetadataReferences(zidx, zi)
	ms.updateWords(zidx, zi)

	// Check if zi must be inserted into ms.idx
	if !ziExist && !zi.isEmpty() {
		ms.idx[zidx.Zid] = zi
	}

	return toCheck
}

func (ms *memStore) updateDeadReferences(zidx *index.ZettelIndex, zi *zettelIndex) {
	// Must only be called if ms.mx is write-locked!
	drefs := zidx.GetDeadRefs()
	newRefs, remRefs := refsDiff(drefs, zi.dead)
	zi.dead = drefs
	for _, ref := range remRefs {
		ms.dead[ref] = remRef(ms.dead[ref], zidx.Zid)
	}
	for _, ref := range newRefs {
		ms.dead[ref] = addRef(ms.dead[ref], zidx.Zid)
	}
}

func (ms *memStore) updateForwardBackwardReferences(zidx *index.ZettelIndex, zi *zettelIndex) {
	// Must only be called if ms.mx is write-locked!
	brefs := zidx.GetBackRefs()
	newRefs, remRefs := refsDiff(brefs, zi.forward)
	zi.forward = brefs
	for _, ref := range remRefs {
		bzi := ms.getEntry(ref)
		bzi.backward = remRef(bzi.backward, zidx.Zid)
	}
	for _, ref := range newRefs {
		bzi := ms.getEntry(ref)
		bzi.backward = addRef(bzi.backward, zidx.Zid)
	}
}

func (ms *memStore) updateMetadataReferences(zidx *index.ZettelIndex, zi *zettelIndex) {
	// Must only be called if ms.mx is write-locked!
	metarefs := zidx.GetMetaRefs()
	for key, mr := range zi.meta {
		if _, ok := metarefs[key]; ok {
			continue
		}
		ms.removeInverseMeta(zidx.Zid, key, mr.forward)
	}
	if zi.meta == nil {
		zi.meta = make(map[string]metaRefs)
	}
	for key, mrefs := range metarefs {
		mr := zi.meta[key]
		newRefs, remRefs := refsDiff(mrefs, mr.forward)
		mr.forward = mrefs
		zi.meta[key] = mr

		for _, ref := range newRefs {
			bzi := ms.getEntry(ref)
			if bzi.meta == nil {
				bzi.meta = make(map[string]metaRefs)
			}
			bmr := bzi.meta[key]
			bmr.backward = addRef(bmr.backward, zidx.Zid)
			bzi.meta[key] = bmr
		}
		ms.removeInverseMeta(zidx.Zid, key, remRefs)
	}
}

func (ms *memStore) updateWords(zidx *index.ZettelIndex, zi *zettelIndex) {
	// Must only be called if ms.mx is write-locked!
	words := zidx.GetWords()
	newWords, removeWords := words.Diff(zi.words)
	for _, word := range newWords {
		if refs, ok := ms.words[word]; ok {
			ms.words[word] = addRef(refs, zidx.Zid)
			continue
		}
		ms.words[word] = id.Slice{zidx.Zid}
	}
	for _, word := range removeWords {
		refs, ok := ms.words[word]
		if !ok {
			continue
		}
		refs2 := remRef(refs, zidx.Zid)
		if len(refs2) == 0 {
			delete(ms.words, word)
			continue
		}
		ms.words[word] = refs2
	}
	zi.words = words.Words()
}

func (ms *memStore) getEntry(zid id.Zid) *zettelIndex {
	// Must only be called if ms.mx is write-locked!
	if zi, ok := ms.idx[zid]; ok {
		return zi
	}
	zi := &zettelIndex{}
	ms.idx[zid] = zi
	return zi
}

func (ms *memStore) DeleteZettel(ctx context.Context, zid id.Zid) id.Set {
	ms.mx.Lock()
	defer ms.mx.Unlock()

	zi, ok := ms.idx[zid]
	if !ok {
		return nil
	}

	ms.deleteDeadSources(zid, zi)
	toCheck := ms.deleteForwardBackward(zid, zi)
	if len(zi.meta) > 0 {
		for key, mrefs := range zi.meta {
			ms.removeInverseMeta(zid, key, mrefs.forward)
		}
	}
	ms.deleteWords(zid, zi.words)
	delete(ms.idx, zid)
	return toCheck
}

func (ms *memStore) deleteDeadSources(zid id.Zid, zi *zettelIndex) {
	// Must only be called if ms.mx is write-locked!
	for _, ref := range zi.dead {
		if drefs, ok := ms.dead[ref]; ok {
			drefs = remRef(drefs, zid)
			if len(drefs) > 0 {
				ms.dead[ref] = drefs
			} else {
				delete(ms.dead, ref)
			}
		}
	}
}

func (ms *memStore) deleteForwardBackward(zid id.Zid, zi *zettelIndex) id.Set {
	// Must only be called if ms.mx is write-locked!
	var toCheck id.Set
	for _, ref := range zi.forward {
		if fzi, ok := ms.idx[ref]; ok {
			fzi.backward = remRef(fzi.backward, zid)
		}
	}
	for _, ref := range zi.backward {
		if bzi, ok := ms.idx[ref]; ok {
			bzi.forward = remRef(bzi.forward, zid)
			if toCheck == nil {
				toCheck = id.NewSet()
			}
			toCheck[ref] = true
		}
	}
	return toCheck
}

func (ms *memStore) removeInverseMeta(zid id.Zid, key string, forward id.Slice) {
	// Must only be called if ms.mx is write-locked!
	for _, ref := range forward {
		bzi, ok := ms.idx[ref]
		if !ok || bzi.meta == nil {
			continue
		}
		bmr, ok := bzi.meta[key]
		if !ok {
			continue
		}
		bmr.backward = remRef(bmr.backward, zid)
		if len(bmr.backward) > 0 || len(bmr.forward) > 0 {
			bzi.meta[key] = bmr
		} else {
			delete(bzi.meta, key)
			if len(bzi.meta) == 0 {
				bzi.meta = nil
			}
		}
	}
}

func (ms *memStore) deleteWords(zid id.Zid, words []string) {
	// Must only be called if ms.mx is write-locked!
	for _, word := range words {
		refs, ok := ms.words[word]
		if !ok {
			continue
		}
		refs2 := remRef(refs, zid)
		if len(refs2) == 0 {
			delete(ms.words, word)
			continue
		}
		ms.words[word] = refs2
	}
}

func (ms *memStore) ReadStats(st *index.StoreStats) {
	ms.mx.RLock()
	st.Zettel = len(ms.idx)
	st.Updates = ms.updates
	st.Words = uint64(len(ms.words))
	ms.mx.RUnlock()
}

func (ms *memStore) Write(w io.Writer) {
	ms.mx.RLock()
	defer ms.mx.RUnlock()

	zids := make(id.Slice, 0, len(ms.idx))
	for id := range ms.idx {
		zids = append(zids, id)
	}
	zids.Sort()
	for _, id := range zids {
		fmt.Fprintln(w, id)
		zi := ms.idx[id]
		fmt.Fprintln(w, "-", zi.dead)
		writeZidsLn(w, ">", zi.forward)
		writeZidsLn(w, "<", zi.backward)
		if zi.meta == nil {
			fmt.Fprintln(w, "*NIL")
		} else if len(zi.meta) == 0 {
			fmt.Fprintln(w, "*(0)")
		} else {
			for k, fb := range zi.meta {
				fmt.Fprintln(w, "*", k)
				writeZidsLn(w, "]", fb.forward)
				writeZidsLn(w, "[", fb.backward)
			}
		}
	}

	zids = make(id.Slice, 0, len(ms.dead))
	for id := range ms.dead {
		zids = append(zids, id)
	}
	zids.Sort()
	for _, id := range zids {
		fmt.Fprintln(w, "~", id, ms.dead[id])
	}
}

func writeZidsLn(w io.Writer, prefix string, zids id.Slice) {
	io.WriteString(w, prefix)
	for _, zid := range zids {
		io.WriteString(w, " ")
		w.Write(zid.Bytes())
	}
	fmt.Fprintln(w)
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package memstore stored the index in main memory.
package memstore

import (
	"zettelstore.de/z/domain/id"
)

func refsDiff(refsN, refsO id.Slice) (newRefs, remRefs id.Slice) {
	npos, opos := 0, 0
	for npos < len(refsN) && opos < len(refsO) {
		rn, ro := refsN[npos], refsO[opos]
		if rn == ro {
			npos++
			opos++
			continue
		}
		if rn < ro {
			newRefs = append(newRefs, rn)
			npos++
			continue
		}
		remRefs = append(remRefs, ro)
		opos++
	}
	if npos < len(refsN) {
		newRefs = append(newRefs, refsN[npos:]...)
	}
	if opos < len(refsO) {
		remRefs = append(remRefs, refsO[opos:]...)
	}
	return newRefs, remRefs
}

func addRef(refs id.Slice, ref id.Zid) id.Slice {
	if len(refs) == 0 {
		return append(refs, ref)
	}
	for i, r := range refs {
		if r == ref {
			return refs
		}
		if r > ref {
			return append(refs[:i], append(id.Slice{ref}, refs[i:]...)...)
		}
	}
	return append(refs, ref)
}

func remRefs(refs id.Slice, rem id.Slice) id.Slice {
	if len(refs) == 0 || len(rem) == 0 {
		return refs
	}
	result := make(id.Slice, 0, len(refs))
	rpos, dpos := 0, 0
	for rpos < len(refs) && dpos < len(rem) {
		rr, dr := refs[rpos], rem[dpos]
		if rr < dr {
			result = append(result, rr)
			rpos++
			continue
		}
		if dr < rr {
			dpos++
			continue
		}
		rpos++
		dpos++
	}
	if rpos < len(refs) {
		result = append(result, refs[rpos:]...)
	}
	return result
}

func remRef(refs id.Slice, ref id.Zid) id.Slice {
	for i, r := range refs {
		if r == ref {
			return append(refs[:i], refs[i+1:]...)
		}
		if r > ref {
			return refs
		}
	}
	return refs
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package memstore stored the index in main memory.
package memstore

import (
	"testing"

	"zettelstore.de/z/domain/id"
)

func assertRefs(t *testing.T, i int, got, exp id.Slice) {
	t.Helper()
	if got == nil && exp != nil {
		t.Errorf("%d: got nil, but expected %v", i, exp)
		return
	}
	if got != nil && exp == nil {
		t.Errorf("%d: expected nil, but got %v", i, got)
		return
	}
	if len(got) != len(exp) {
		t.Errorf("%d: expected len(%v)==%d, but got len(%v)==%d", i, exp, len(exp), got, len(got))
		return
	}
	for p, n := range exp {
		if got := got[p]; got != id.Zid(n) {
			t.Errorf("%d: pos %d: expected %d, but got %d", i, p, n, got)
		}
	}
}

func TestRefsDiff(t *testing.T) {
	testcases := []struct {
		in1, in2   id.Slice
		exp1, exp2 id.Slice
	}{
		{nil, nil, nil, nil},
		{id.Slice{1}, nil, id.Slice{1}, nil},
		{nil, id.Slice{1}, nil, id.Slice{1}},
		{id.Slice{1}, id.Slice{1}, nil, nil},
		{id.Slice{1, 2}, id.Slice{1}, id.Slice{2}, nil},
		{id.Slice{1, 2}, id.Slice{1, 3}, id.Slice{2}, id.Slice{3}},
		{id.Slice{1, 4}, id.Slice{1, 3}, id.Slice{4}, id.Slice{3}},
	}
	for i, tc := range testcases {
		got1, got2 := refsDiff(tc.in1, tc.in2)
		assertRefs(t, i, got1, tc.exp1)
		assertRefs(t, i, got2, tc.exp2)
	}
}

func TestAddRef(t *testing.T) {
	testcases := []struct {
		ref id.Slice
		zid uint
		exp id.Slice
	}{
		{nil, 5, id.Slice{5}},
		{id.Slice{1}, 5, id.Slice{1, 5}},
		{id.Slice{10}, 5, id.Slice{5, 10}},
		{id.Slice{5}, 5, id.Slice{5}},
		{id.Slice{1, 10}, 5, id.Slice{1, 5, 10}},
		{id.Slice{1, 5, 10}, 5, id.Slice{1, 5, 10}},
	}
	for i, tc := range testcases {
		got := addRef(tc.ref, id.Zid(tc.zid))
		assertRefs(t, i, got, tc.exp)
	}
}

func TestRemRefs(t *testing.T) {
	testcases := []struct {
		in1, in2 id.Slice
		exp      id.Slice
	}{
		{nil, nil, nil},
		{nil, id.Slice{}, nil},
		{id.Slice{}, nil, id.Slice{}},
		{id.Slice{}, id.Slice{}, id.Slice{}},
		{id.Slice{1}, id.Slice{5}, id.Slice{1}},
		{id.Slice{10}, id.Slice{5}, id.Slice{10}},
		{id.Slice{1, 5}, id.Slice{5}, id.Slice{1}},
		{id.Slice{5, 10}, id.Slice{5}, id.Slice{10}},
		{id.Slice{1, 10}, id.Slice{5}, id.Slice{1, 10}},
		{id.Slice{1}, id.Slice{2, 5}, id.Slice{1}},
		{id.Slice{10}, id.Slice{2, 5}, id.Slice{10}},
		{id.Slice{1, 5}, id.Slice{2, 5}, id.Slice{1}},
		{id.Slice{5, 10}, id.Slice{2, 5}, id.Slice{10}},
		{id.Slice{1, 2, 5}, id.Slice{2, 5}, id.Slice{1}},
		{id.Slice{2, 5, 10}, id.Slice{2, 5}, id.Slice{10}},
		{id.Slice{1, 10}, id.Slice{2, 5}, id.Slice{1, 10}},
		{id.Slice{1}, id.Slice{5, 9}, id.Slice{1}},
		{id.Slice{10}, id.Slice{5, 9}, id.Slice{10}},
		{id.Slice{1, 5}, id.Slice{5, 9}, id.Slice{1}},
		{id.Slice{5, 10}, id.Slice{5, 9}, id.Slice{10}},
		{id.Slice{1, 5, 9}, id.Slice{5, 9}, id.Slice{1}},
		{id.Slice{5, 9, 10}, id.Slice{5, 9}, id.Slice{10}},
		{id.Slice{1, 10}, id.Slice{5, 9}, id.Slice{1, 10}},
	}
	for i, tc := range testcases {
		got := remRefs(tc.in1, tc.in2)
		assertRefs(t, i, got, tc.exp)
	}
}

func TestRemRef(t *testing.T) {
	testcases := []struct {
		ref id.Slice
		zid uint
		exp id.Slice
	}{
		{nil, 5, nil},
		{id.Slice{}, 5, id.Slice{}},
		{id.Slice{5}, 5, id.Slice{}},
		{id.Slice{1}, 5, id.Slice{1}},
		{id.Slice{10}, 5, id.Slice{10}},
		{id.Slice{1, 5}, 5, id.Slice{1}},
		{id.Slice{5, 10}, 5, id.Slice{10}},
		{id.Slice{1, 5, 10}, 5, id.Slice{1, 10}},
	}
	for i, tc := range testcases {
		got := remRef(tc.ref, id.Zid(tc.zid))
		assertRefs(t, i, got, tc.exp)
	}
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package index allows to search for metadata and content.
package index

// WordSet contains the set of all words, with the count of their occurrences.
type WordSet map[string]int

// Words gives the slice of all words in the set.
func (ws WordSet) Words() []string {
	if len(ws) == 0 {
		return nil
	}
	words := make([]string, 0, len(ws))
	for w := range ws {
		words = append(words, w)
	}
	return words
}

// Diff calculates the word slice to be added and to be removed from oldWords
// to get the given word set.
func (ws WordSet) Diff(oldWords []string) (newWords, removeWords []string) {
	if len(ws) == 0 {
		return nil, oldWords
	}
	if len(oldWords) == 0 {
		return ws.Words(), nil
	}
	oldSet := make(WordSet, len(oldWords))
	for _, ow := range oldWords {
		if _, ok := ws[ow]; ok {
			oldSet[ow] = 1
			continue
		}
		removeWords = append(removeWords, ow)
	}
	for w := range ws {
		if _, ok := oldSet[w]; ok {
			continue
		}
		newWords = append(newWords, w)
	}
	return newWords, removeWords
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package index allows to search for metadata and content.
package index_test

import (
	"sort"
	"testing"

	"zettelstore.de/z/index"
)

func equalWordList(exp, got []string) bool {
	if len(exp) != len(got) {
		return false
	}
	if len(got) == 0 {
		return len(exp) == 0
	}
	sort.Strings(got)
	for i, w := range exp {
		if w != got[i] {
			return false
		}
	}
	return true
}

func TestWordsWords(t *testing.T) {
	testcases := []struct {
		words index.WordSet
		exp   []string
	}{
		{nil, nil},
		{index.WordSet{}, nil},
		{index.WordSet{"a": 1, "b": 2}, []string{"a", "b"}},
	}
	for i, tc := range testcases {
		got := tc.words.Words()
		if !equalWordList(tc.exp, got) {
			t.Errorf("%d: %v.Words() == %v, but got %v", i, tc.words, tc.exp, got)
		}
	}
}

func TestWordsDiff(t *testing.T) {
	testcases := []struct {
		cur        index.WordSet
		old        []string
		expN, expR []string
	}{
		{nil, nil, nil, nil},
		{index.WordSet{}, []string{}, nil, nil},
		{index.WordSet{"a": 1}, []string{}, []string{"a"}, nil},
		{index.WordSet{"a": 1}, []string{"b"}, []string{"a"}, []string{"b"}},
		{index.WordSet{}, []string{"b"}, nil, []string{"b"}},
		{index.WordSet{"a": 1}, []string{"a"}, nil, nil},
	}
	for i, tc := range testcases {
		gotN, gotR := tc.cur.Diff(tc.old)
		if !equalWordList(tc.expN, gotN) {
			t.Errorf("%d: %v.Diff(%v)->new %v, but got %v", i, tc.cur, tc.old, tc.expN, gotN)
		}
		if !equalWordList(tc.expR, gotR) {
			t.Errorf("%d: %v.Diff(%v)->rem %v, but got %v", i, tc.cur, tc.old, tc.expR, gotR)
		}
	}
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021 Detlef Stern
//
// This file is part of zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//-----------------------------------------------------------------------------

// Package index allows to search for metadata and content.
package index

import (
	"zettelstore.de/z/domain/id"
)

// ZettelIndex contains all index data of a zettel.
type ZettelIndex struct {
	Zid      id.Zid            // zid of the indexed zettel
	backrefs id.Set            // set of back references
	metarefs map[string]id.Set // references to inverse keys
	deadrefs id.Set            // set of dead references
	words    WordSet
}

// NewZettelIndex creates a new zettel index.
func NewZettelIndex(zid id.Zid) *ZettelIndex {
	return &ZettelIndex{
		Zid:      zid,
		backrefs: id.NewSet(),
		metarefs: make(map[string]id.Set),
		deadrefs: id.NewSet(),
	}
}

// AddBackRef adds a reference to a zettel where the current zettel links to
// without any more information.
func (zi *ZettelIndex) AddBackRef(zid id.Zid) {
	zi.backrefs[zid] = true
}

// AddMetaRef adds a named reference to a zettel. On that zettel, the given
// metadata key should point back to the current zettel.
func (zi *ZettelIndex) AddMetaRef(key string, zid id.Zid) {
	if zids, ok := zi.metarefs[key]; ok {
		zids[zid] = true
		return
	}
	zi.metarefs[key] = id.NewSet(zid)
}

// AddDeadRef adds a dead reference to a zettel.
func (zi *ZettelIndex) AddDeadRef(zid id.Zid) {
	zi.deadrefs[zid] = true
}

// SetWords sets the words to the given value.
func (zi *ZettelIndex) SetWords(words WordSet) { zi.words = words }

// GetDeadRefs returns all dead references as a sorted list.
func (zi *ZettelIndex) GetDeadRefs() id.Slice {
	return zi.deadrefs.Sorted()
}

// GetBackRefs returns all back references as a sorted list.
func (zi *ZettelIndex) GetBackRefs() id.Slice {
	return zi.backrefs.Sorted()
}

// GetMetaRefs returns all meta references as a map of strings to a sorted list of references
func (zi *ZettelIndex) GetMetaRefs() map[string]id.Slice {
	if len(zi.metarefs) == 0 {
		return nil
	}
	result := make(map[string]id.Slice, len(zi.metarefs))
	for key, refs := range zi.metarefs {
		result[key] = refs.Sorted()
	}
	return result
}

// GetWords returns a reference to the WordSet. It must not be modified.
func (zi *ZettelIndex) GetWords() WordSet { return zi.words }