// SPDX-License-Identifier: EUPL-1.2
// SPDX-FileCopyrightText: 2020-present Detlef Stern
//-----------------------------------------------------------------------------

package ast

import (
	"unicode/utf8"

	"t73f.de/r/zsc/attrs"
)

// Definitions of inline nodes.

// InlineSlice is a list of BlockNodes.
type InlineSlice []InlineNode

func (*InlineSlice) inlineNode() { /* Just a marker */ }

// CreateInlineSliceFromWords makes a new inline list from words,
// that will be space-separated.
func CreateInlineSliceFromWords(words ...string) InlineSlice {
	inl := make(InlineSlice, 0, 2*len(words)-1)
	for i, word := range words {
		if i > 0 {
			inl = append(inl, &SpaceNode{Lexeme: " "})
		}
		inl = append(inl, &TextNode{Text: word})
	}
	return inl
}

// WalkChildren walks down to the list.
func (is *InlineSlice) WalkChildren(v Visitor) {
	for _, in := range *is {
		Walk(v, in)
	}
}

// --------------------------------------------------------------------------

// TextNode just contains some text.
type TextNode struct {
	Text string // The text itself.
}

func (*TextNode) inlineNode() { /* Just a marker */ }

// WalkChildren does nothing.
func (*TextNode) WalkChildren(Visitor) { /* No children*/ }

// --------------------------------------------------------------------------

// SpaceNode tracks inter-word space characters.
type SpaceNode struct {
	Lexeme string
}

func (*SpaceNode) inlineNode() { /* Just a marker */ }

// WalkChildren does nothing.
func (*SpaceNode) WalkChildren(Visitor) { /* No children*/ }

// Count returns the number of space runes.
func (sn *SpaceNode) Count() int {
	return utf8.RuneCountInString(sn.Lexeme)
}

// --------------------------------------------------------------------------

// BreakNode signals a new line that must / should be interpreted as a new line break.
type BreakNode struct {
	Hard bool // Hard line break?
}

	"t73f.de/r/zsc/attrs"
	"zettelstore.de/z/ast"
)

func BenchmarkWalk(b *testing.B) {
	root := ast.BlockSlice{
		&ast.HeadingNode{
			Inlines: ast.CreateInlineSliceFromWords("A", "Simple", "Heading"),
		},
		&ast.ParaNode{
			Inlines: ast.CreateInlineSliceFromWords("This", "is", "the", "introduction."),
		},
		&ast.NestedListNode{
			Kind: ast.NestedListUnordered,
			Items: []ast.ItemSlice{
				[]ast.ItemNode{
					&ast.ParaNode{
						Inlines: ast.CreateInlineSliceFromWords("Item", "1"),
					},
				},
				[]ast.ItemNode{
					&ast.ParaNode{
						Inlines: ast.CreateInlineSliceFromWords("Item", "2"),
					},
				},
			},
		},
		&ast.ParaNode{
			Inlines: ast.CreateInlineSliceFromWords("This", "is", "some", "intermediate", "text."),
		},
		ast.CreateParaNode(
			&ast.FormatNode{
				Kind: ast.FormatEmph,
				Attrs: attrs.Attributes(map[string]string{
					"":      "class",
					"color": "green",
				}),
				Inlines: ast.CreateInlineSliceFromWords("This", "is", "some", "emphasized", "text."),
			},
			&ast.SpaceNode{Lexeme: " "},
			&ast.LinkNode{
				Ref:     &ast.Reference{Value: "http://zettelstore.de"},
				Inlines: ast.CreateInlineSliceFromWords("URL", "text."),
			},
		),
	}
	v := benchVisitor{}
	b.ResetTimer()
	for range b.N {
		ast.Walk(&v, &root)

	return zettel.Zettel{}, box.NewErrNotAllowed("GetZettel", user, zid)
}

func (pp *polBox) GetAllZettel(ctx context.Context, zid id.Zid) ([]zettel.Zettel, error) {
	return pp.box.GetAllZettel(ctx, zid)
}

func (pp *polBox) FetchZids(ctx context.Context) (id.Set, error) {
	return nil, box.NewErrNotAllowed("fetch-zids", server.GetUser(ctx), id.Invalid)
}

func (pp *polBox) GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) {
	m, err := pp.box.GetMeta(ctx, zid)
	if err != nil {
		return nil, err

// Box is to be used outside the box package and its descendants.
type Box interface {
	BaseBox
	WriteBox

	// FetchZids returns the set of all zettel identifer managed by the box.
	FetchZids(ctx context.Context) (id.Set, error)

	// GetMeta returns the metadata of the zettel with the given identifier.
	GetMeta(context.Context, id.Zid) (*meta.Meta, error)

	// SelectMeta returns a list of metadata that comply to the given selection criteria.
	// If `metaSeq` is `nil`, the box assumes metadata of all available zettel.
	SelectMeta(ctx context.Context, metaSeq []*meta.Meta, q *query.Query) ([]*meta.Meta, error)

	arNothing arAction = iota
	arReload
	arZettel
)

type anteroom struct {
	next    *anteroom
	waiting id.Set
	curLoad int
	reload  bool
}

type anteroomQueue struct {
	mx      sync.Mutex
	first   *anteroom

		ar.last = ar.first
		return
	}
	for room := ar.first; room != nil; room = room.next {
		if room.reload {
			continue // Do not put zettel in reload room
		}
		if _, ok := room.waiting[zid]; ok {
			// Zettel is already waiting. Nothing to do.
			return
		}
	}
	if room := ar.last; !room.reload && (ar.maxLoad == 0 || room.curLoad < ar.maxLoad) {
		room.waiting.Add(zid)
		room.curLoad++

func (ar *anteroomQueue) Reset() {
	ar.mx.Lock()
	defer ar.mx.Unlock()
	ar.first = &anteroom{next: nil, waiting: nil, curLoad: 0, reload: true}
	ar.last = ar.first
}

func (ar *anteroomQueue) Reload(allZids id.Set) {
	ar.mx.Lock()
	defer ar.mx.Unlock()
	ar.deleteReloadedRooms()

	if ns := len(allZids); ns > 0 {
		ar.first = &anteroom{next: ar.first, waiting: allZids, curLoad: ns, reload: true}
		if ar.first.next == nil {
			ar.last = ar.first
		}
	} else {
		ar.first = nil
		ar.last = nil
	}

	defer ar.mx.Unlock()
	first := ar.first
	if first != nil {
		if first.waiting == nil && first.reload {
			ar.removeFirst()
			return arReload, id.Invalid, false
		}
		for zid := range first.waiting {
			delete(first.waiting, zid)
			if len(first.waiting) == 0 {
				ar.removeFirst()
			}
			return arZettel, zid, first.reload
		}
		ar.removeFirst()
	}
	return arNothing, id.Invalid, false

			result = append(result, z)
		}
	}
	return result, nil
}

// FetchZids returns the set of all zettel identifer managed by the box.
func (mgr *Manager) FetchZids(ctx context.Context) (id.Set, error) {
	mgr.mgrLog.Debug().Msg("FetchZids")
	if mgr.State() != box.StartStateStarted {
		return nil, box.ErrStopped
	}
	result := id.Set{}
	mgr.mgrMx.RLock()
	defer mgr.mgrMx.RUnlock()
	for _, p := range mgr.boxes {
		err := p.ApplyZid(ctx, func(zid id.Zid) { result.Add(zid) }, func(id.Zid) bool { return true })
		if err != nil {
			return nil, err
		}

	compSearch := q.RetrieveAndCompile(ctx, mgr, metaSeq)
	if result := compSearch.Result(); result != nil {
		mgr.mgrLog.Trace().Int("count", int64(len(result))).Msg("found without ApplyMeta")
		return result, nil
	}
	selected := map[id.Zid]*meta.Meta{}
	for _, term := range compSearch.Terms {
		rejected := id.Set{}
		handleMeta := func(m *meta.Meta) {
			zid := m.Zid
			if rejected.ContainsOrNil(zid) {
				mgr.mgrLog.Trace().Zid(zid).Msg("SelectMeta/alreadyRejected")
				return
			}
			if _, ok := selected[zid]; ok {
				mgr.mgrLog.Trace().Zid(zid).Msg("SelectMeta/alreadySelected")
				return
			}

	"zettelstore.de/z/ast"
	"zettelstore.de/z/box/manager/store"
	"zettelstore.de/z/strfun"
	"zettelstore.de/z/zettel/id"
)

type collectData struct {
	refs  id.Set
	words store.WordSet
	urls  store.WordSet
}

func (data *collectData) initialize() {
	data.refs = id.NewSet()
	data.words = store.NewWordSet()

	"zettelstore.de/z/zettel"
	"zettelstore.de/z/zettel/id"
	"zettelstore.de/z/zettel/meta"
)

// SearchEqual returns all zettel that contains the given exact word.
// The word must be normalized through Unicode NKFD, trimmed and not empty.
func (mgr *Manager) SearchEqual(word string) id.Set {
	found := mgr.idxStore.SearchEqual(word)
	mgr.idxLog.Debug().Str("word", word).Int("found", int64(len(found))).Msg("SearchEqual")
	if msg := mgr.idxLog.Trace(); msg.Enabled() {
		msg.Str("ids", fmt.Sprint(found)).Msg("IDs")
	}
	return found
}

// SearchPrefix returns all zettel that have a word with the given prefix.
// The prefix must be normalized through Unicode NKFD, trimmed and not empty.
func (mgr *Manager) SearchPrefix(prefix string) id.Set {
	found := mgr.idxStore.SearchPrefix(prefix)
	mgr.idxLog.Debug().Str("prefix", prefix).Int("found", int64(len(found))).Msg("SearchPrefix")
	if msg := mgr.idxLog.Trace(); msg.Enabled() {
		msg.Str("ids", fmt.Sprint(found)).Msg("IDs")
	}
	return found
}

// SearchSuffix returns all zettel that have a word with the given suffix.
// The suffix must be normalized through Unicode NKFD, trimmed and not empty.
func (mgr *Manager) SearchSuffix(suffix string) id.Set {
	found := mgr.idxStore.SearchSuffix(suffix)
	mgr.idxLog.Debug().Str("suffix", suffix).Int("found", int64(len(found))).Msg("SearchSuffix")
	if msg := mgr.idxLog.Trace(); msg.Enabled() {
		msg.Str("ids", fmt.Sprint(found)).Msg("IDs")
	}
	return found
}

// SearchContains returns all zettel that contains the given string.
// The string must be normalized through Unicode NKFD, trimmed and not empty.
func (mgr *Manager) SearchContains(s string) id.Set {
	found := mgr.idxStore.SearchContains(s)
	mgr.idxLog.Debug().Str("s", s).Int("found", int64(len(found))).Msg("SearchContains")
	if msg := mgr.idxLog.Trace(); msg.Enabled() {
		msg.Str("ids", fmt.Sprint(found)).Msg("IDs")
	}
	return found
}

// idxIndexer runs in the background and updates the index data structures.

		if !ok {
			return false
		}
	case _, ok := <-timer.C:
		if !ok {
			return false
		}

		timer.Reset(timerDuration)
	case <-mgr.done:
		if !timer.Stop() {
			<-timer.C
		}
		return false
	}

		}
	} else {
		stWords.Add(value)
	}
}

func (mgr *Manager) idxProcessData(ctx context.Context, zi *store.ZettelIndex, cData *collectData) {
	for ref := range cData.refs {
		if mgr.HasZettel(ctx, ref) {
			zi.AddBackRef(ref)
		} else {
			zi.AddDeadRef(ref)
		}
	}
	zi.SetWords(cData.words)
	zi.SetUrls(cData.urls)
}

func (mgr *Manager) idxUpdateValue(ctx context.Context, inverseKey, value string, zi *store.ZettelIndex) {
	zid, err := id.Parse(value)
	if err != nil {

}

func (mgr *Manager) idxDeleteZettel(ctx context.Context, zid id.Zid) {
	toCheck := mgr.idxStore.DeleteZettel(ctx, zid)
	mgr.idxCheckZettel(toCheck)
}

func (mgr *Manager) idxCheckZettel(s id.Set) {
	for zid := range s {
		mgr.idxAr.EnqueueZettel(zid)
	}
}

	"zettelstore.de/z/box/manager/store"
	"zettelstore.de/z/zettel/id"
	"zettelstore.de/z/zettel/meta"
)

type zettelData struct {
	meta      *meta.Meta // a local copy of the metadata, without computed keys
	dead      id.Slice   // list of dead references in this zettel
	forward   id.Slice   // list of forward references in this zettel
	backward  id.Slice   // list of zettel that reference with zettel
	otherRefs map[string]bidiRefs
	words     []string // list of words of this zettel
	urls      []string // list of urls of this zettel
}

type bidiRefs struct {
	forward  id.Slice
	backward id.Slice
}





type stringRefs map[string]id.Slice





type memStore struct {
	mx     sync.RWMutex
	intern map[string]string // map to intern strings
	idx    map[id.Zid]*zettelData
	dead   map[id.Zid]id.Slice // map dead refs where they occur
	words  stringRefs
	urls   stringRefs

	// Stats
	mxStats sync.Mutex
	updates uint64
}


// New returns a new memory-based index store.
func New() store.Store {
	return &memStore{
		intern: make(map[string]string, 1024),
		idx:    make(map[id.Zid]*zettelData),
		dead:   make(map[id.Zid]id.Slice),
		words:  make(stringRefs),
		urls:   make(stringRefs),
	}
}

func (ms *memStore) GetMeta(_ context.Context, zid id.Zid) (*meta.Meta, error) {
	ms.mx.RLock()
	defer ms.mx.RUnlock()
	if zi, found := ms.idx[zid]; found && zi.meta != nil {
		// zi.meta is nil, if zettel was referenced, but is not indexed yet.
		return zi.meta.Clone(), nil
	}
	return nil, box.ErrZettelNotFound{Zid: zid}
}

func (ms *memStore) Enrich(_ context.Context, m *meta.Meta) {
	if ms.doEnrich(m) {
		ms.mxStats.Lock()
		ms.updates++
		ms.mxStats.Unlock()
	}
}

func (ms *memStore) doEnrich(m *meta.Meta) bool {
	ms.mx.RLock()
	defer ms.mx.RUnlock()
	zi, ok := ms.idx[m.Zid]
	if !ok {
		return false
	}
	var updated bool
	if len(zi.dead) > 0 {
		m.Set(api.KeyDead, zi.dead.String())
		updated = true
	}
	back := removeOtherMetaRefs(m, zi.backward.Clone())
	if len(zi.backward) > 0 {
		m.Set(api.KeyBackward, zi.backward.String())
		updated = true
	}
	if len(zi.forward) > 0 {
		m.Set(api.KeyForward, zi.forward.String())
		back = remRefs(back, zi.forward)
		updated = true
	}
	for k, refs := range zi.otherRefs {
		if len(refs.backward) > 0 {
			m.Set(k, refs.backward.String())
			back = remRefs(back, refs.backward)
			updated = true
		}
	}
	if len(back) > 0 {
		m.Set(api.KeyBack, back.String())
		updated = true
	}
	return updated
}

// SearchEqual returns all zettel that contains the given exact word.
// The word must be normalized through Unicode NKFD, trimmed and not empty.
func (ms *memStore) SearchEqual(word string) id.Set {
	ms.mx.RLock()
	defer ms.mx.RUnlock()
	result := id.NewSet()
	if refs, ok := ms.words[word]; ok {
		result.CopySlice(refs)
	}
	if refs, ok := ms.urls[word]; ok {
		result.CopySlice(refs)
	}
	zid, err := id.Parse(word)
	if err != nil {
		return result
	}
	zi, ok := ms.idx[zid]
	if !ok {
		return result
	}

	addBackwardZids(result, zid, zi)
	return result
}

// SearchPrefix returns all zettel that have a word with the given prefix.
// The prefix must be normalized through Unicode NKFD, trimmed and not empty.
func (ms *memStore) SearchPrefix(prefix string) id.Set {
	ms.mx.RLock()
	defer ms.mx.RUnlock()
	result := ms.selectWithPred(prefix, strings.HasPrefix)
	l := len(prefix)
	if l > 14 {
		return result
	}

		minZid, err = id.Parse(prefix + "00000000000000"[:14-l])
		if err != nil {
			return result
		}
	}
	for zid, zi := range ms.idx {
		if minZid <= zid && zid <= maxZid {
			addBackwardZids(result, zid, zi)
		}
	}
	return result
}

// SearchSuffix returns all zettel that have a word with the given suffix.
// The suffix must be normalized through Unicode NKFD, trimmed and not empty.
func (ms *memStore) SearchSuffix(suffix string) id.Set {
	ms.mx.RLock()
	defer ms.mx.RUnlock()
	result := ms.selectWithPred(suffix, strings.HasSuffix)
	l := len(suffix)
	if l > 14 {
		return result
	}
	val, err := id.ParseUint(suffix)
	if err != nil {
		return result
	}
	modulo := uint64(1)
	for range l {
		modulo *= 10
	}
	for zid, zi := range ms.idx {
		if uint64(zid)%modulo == val {
			addBackwardZids(result, zid, zi)
		}
	}
	return result
}

// SearchContains returns all zettel that contains the given string.
// The string must be normalized through Unicode NKFD, trimmed and not empty.
func (ms *memStore) SearchContains(s string) id.Set {
	ms.mx.RLock()
	defer ms.mx.RUnlock()
	result := ms.selectWithPred(s, strings.Contains)
	if len(s) > 14 {
		return result
	}
	if _, err := id.ParseUint(s); err != nil {
		return result
	}
	for zid, zi := range ms.idx {
		if strings.Contains(zid.String(), s) {
			addBackwardZids(result, zid, zi)
		}
	}
	return result
}

func (ms *memStore) selectWithPred(s string, pred func(string, string) bool) id.Set {
	// Must only be called if ms.mx is read-locked!
	result := id.NewSet()
	for word, refs := range ms.words {
		if !pred(word, s) {
			continue
		}
		result.CopySlice(refs)
	}
	for u, refs := range ms.urls {
		if !pred(u, s) {
			continue
		}
		result.CopySlice(refs)










	}
	return result
}

func addBackwardZids(result id.Set, zid id.Zid, zi *zettelData) {
	// Must only be called if ms.mx is read-locked!
	result.Add(zid)
	result.CopySlice(zi.backward)
	for _, mref := range zi.otherRefs {
		result.CopySlice(mref.backward)
	}
}

func removeOtherMetaRefs(m *meta.Meta, back id.Slice) id.Slice {
	for _, p := range m.PairsRest() {
		switch meta.Type(p.Key) {
		case meta.TypeID:
			if zid, err := id.Parse(p.Value); err == nil {
				back = remRef(back, zid)
			}
		case meta.TypeIDSet:
			for _, val := range meta.ListFromValue(p.Value) {
				if zid, err := id.Parse(val); err == nil {
					back = remRef(back, zid)
				}
			}
		}
	}
	return back
}

func (ms *memStore) UpdateReferences(_ context.Context, zidx *store.ZettelIndex) id.Set {
	ms.mx.Lock()
	defer ms.mx.Unlock()
	m := ms.makeMeta(zidx)
	zi, ziExist := ms.idx[zidx.Zid]
	if !ziExist || zi == nil {
		zi = &zettelData{}
		ziExist = false
	}

	// Is this zettel an old dead reference mentioned in other zettel?
	var toCheck id.Set
	if refs, ok := ms.dead[zidx.Zid]; ok {
		// These must be checked later again
		toCheck = id.NewSet(refs...)
		delete(ms.dead, zidx.Zid)
	}

	zi.meta = m
	ms.updateDeadReferences(zidx, zi)
	ids := ms.updateForwardBackwardReferences(zidx, zi)
	toCheck = toCheck.Copy(ids)
	ids = ms.updateMetadataReferences(zidx, zi)
	toCheck = toCheck.Copy(ids)
	zi.words = updateStrings(zidx.Zid, ms.words, zi.words, zidx.GetWords())
	zi.urls = updateStrings(zidx.Zid, ms.urls, zi.urls, zidx.GetUrls())

	// Check if zi must be inserted into ms.idx
	if !ziExist {
		ms.idx[zidx.Zid] = zi
	}

	return toCheck
}

var internableKeys = map[string]bool{
	api.KeyRole:      true,
	api.KeySyntax:    true,
	api.KeyFolgeRole: true,
	api.KeyLang:      true,
	api.KeyReadOnly:  true,
}

func isInternableValue(key string) bool {
	if internableKeys[key] {
		return true
	}
	return strings.HasSuffix(key, meta.SuffixKeyRole)
}

func (ms *memStore) internString(s string) string {
	if is, found := ms.intern[s]; found {
		return is
	}
	ms.intern[s] = s
	return s
}

func (ms *memStore) makeMeta(zidx *store.ZettelIndex) *meta.Meta {
	origM := zidx.GetMeta()
	copyM := meta.New(origM.Zid)
	for _, p := range origM.Pairs() {
		key := ms.internString(p.Key)
		if isInternableValue(key) {
			copyM.Set(key, ms.internString(p.Value))
		} else if key == api.KeyBoxNumber || !meta.IsComputed(key) {
			copyM.Set(key, p.Value)
		}
	}
	return copyM
}

func (ms *memStore) updateDeadReferences(zidx *store.ZettelIndex, zi *zettelData) {
	// Must only be called if ms.mx is write-locked!
	drefs := zidx.GetDeadRefs()
	newRefs, remRefs := refsDiff(drefs, zi.dead)
	zi.dead = drefs
	for _, ref := range remRefs {
		ms.dead[ref] = remRef(ms.dead[ref], zidx.Zid)
	}
	for _, ref := range newRefs {
		ms.dead[ref] = addRef(ms.dead[ref], zidx.Zid)
	}
}

func (ms *memStore) updateForwardBackwardReferences(zidx *store.ZettelIndex, zi *zettelData) id.Set {
	// Must only be called if ms.mx is write-locked!
	brefs := zidx.GetBackRefs()

	newRefs, remRefs := refsDiff(brefs, zi.forward)
	zi.forward = brefs

	var toCheck id.Set
	for _, ref := range remRefs {
		bzi := ms.getOrCreateEntry(ref)
		bzi.backward = remRef(bzi.backward, zidx.Zid)
		if bzi.meta == nil {
			toCheck = toCheck.Add(ref)
		}
	}
	for _, ref := range newRefs {
		bzi := ms.getOrCreateEntry(ref)
		bzi.backward = addRef(bzi.backward, zidx.Zid)
		if bzi.meta == nil {
			toCheck = toCheck.Add(ref)
		}
	}
	return toCheck
}

func (ms *memStore) updateMetadataReferences(zidx *store.ZettelIndex, zi *zettelData) id.Set {
	// Must only be called if ms.mx is write-locked!
	inverseRefs := zidx.GetInverseRefs()
	for key, mr := range zi.otherRefs {
		if _, ok := inverseRefs[key]; ok {
			continue
		}
		ms.removeInverseMeta(zidx.Zid, key, mr.forward)
	}
	if zi.otherRefs == nil {
		zi.otherRefs = make(map[string]bidiRefs)
	}
	var toCheck id.Set
	for key, mrefs := range inverseRefs {
		mr := zi.otherRefs[key]
		newRefs, remRefs := refsDiff(mrefs, mr.forward)
		mr.forward = mrefs
		zi.otherRefs[key] = mr

		for _, ref := range newRefs {
			bzi := ms.getOrCreateEntry(ref)
			if bzi.otherRefs == nil {
				bzi.otherRefs = make(map[string]bidiRefs)
			}
			bmr := bzi.otherRefs[key]
			bmr.backward = addRef(bmr.backward, zidx.Zid)
			bzi.otherRefs[key] = bmr
			if bzi.meta == nil {
				toCheck = toCheck.Add(ref)
			}

		}
		ms.removeInverseMeta(zidx.Zid, key, remRefs)
	}
	return toCheck
}

func updateStrings(zid id.Zid, srefs stringRefs, prev []string, next store.WordSet) []string {
	newWords, removeWords := next.Diff(prev)
	for _, word := range newWords {
		if refs, ok := srefs[word]; ok {
			srefs[word] = addRef(refs, zid)
			continue
		}
		srefs[word] = id.Slice{zid}
	}
	for _, word := range removeWords {
		refs, ok := srefs[word]
		if !ok {
			continue
		}
		refs2 := remRef(refs, zid)
		if len(refs2) == 0 {
			delete(srefs, word)
			continue
		}
		srefs[word] = refs2
	}
	return next.Words()
}

func (ms *memStore) getOrCreateEntry(zid id.Zid) *zettelData {
	// Must only be called if ms.mx is write-locked!
	if zi, ok := ms.idx[zid]; ok {
		return zi
	}
	zi := &zettelData{}
	ms.idx[zid] = zi
	return zi
}

func (ms *memStore) RenameZettel(_ context.Context, curZid, newZid id.Zid) id.Set {
	ms.mx.Lock()
	defer ms.mx.Unlock()

	curZi, curFound := ms.idx[curZid]
	_, newFound := ms.idx[newZid]
	if !curFound || newFound {
		return nil
	}
	newZi := &zettelData{
		meta:      copyMeta(curZi.meta, newZid),
		dead:      ms.copyDeadReferences(curZi.dead),
		forward:   ms.copyForward(curZi.forward, newZid),
		backward:  nil, // will be done through tocheck
		otherRefs: nil, // TODO: check if this will be done through toCheck
		words:     copyStrings(ms.words, curZi.words, newZid),
		urls:      copyStrings(ms.urls, curZi.urls, newZid),
	}

	ms.idx[newZid] = newZi
	toCheck := ms.doDeleteZettel(curZid)
	toCheck = toCheck.CopySlice(ms.dead[newZid])
	delete(ms.dead, newZid)
	toCheck = toCheck.Add(newZid) // should update otherRefs
	return toCheck
}
func copyMeta(m *meta.Meta, newZid id.Zid) *meta.Meta {
	result := m.Clone()
	result.Zid = newZid
	return result
}
func (ms *memStore) copyDeadReferences(curDead id.Slice) id.Slice {
	// Must only be called if ms.mx is write-locked!
	if l := len(curDead); l > 0 {
		result := make(id.Slice, l)
		for i, ref := range curDead {
			result[i] = ref
			ms.dead[ref] = addRef(ms.dead[ref], ref)
		}
		return result
	}
	return nil
}
func (ms *memStore) copyForward(curForward id.Slice, newZid id.Zid) id.Slice {
	// Must only be called if ms.mx is write-locked!
	if l := len(curForward); l > 0 {
		result := make(id.Slice, l)
		for i, ref := range curForward {
			result[i] = ref
			if fzi, found := ms.idx[ref]; found {
				fzi.backward = addRef(fzi.backward, newZid)
			}
		}
		return result
	}
	return nil
}
func copyStrings(msStringMap stringRefs, curStrings []string, newZid id.Zid) []string {
	// Must only be called if ms.mx is write-locked!
	if l := len(curStrings); l > 0 {
		result := make([]string, l)
		for i, s := range curStrings {
			result[i] = s
			msStringMap[s] = addRef(msStringMap[s], newZid)
		}
		return result
	}
	return nil
}

func (ms *memStore) DeleteZettel(_ context.Context, zid id.Zid) id.Set {
	ms.mx.Lock()
	defer ms.mx.Unlock()
	return ms.doDeleteZettel(zid)
}

func (ms *memStore) doDeleteZettel(zid id.Zid) id.Set {
	// Must only be called if ms.mx is write-locked!
	zi, ok := ms.idx[zid]
	if !ok {
		return nil
	}

	ms.deleteDeadSources(zid, zi)
	toCheck := ms.deleteForwardBackward(zid, zi)
	for key, mrefs := range zi.otherRefs {
		ms.removeInverseMeta(zid, key, mrefs.forward)
	}
	deleteStrings(ms.words, zi.words, zid)
	deleteStrings(ms.urls, zi.urls, zid)
	delete(ms.idx, zid)
	return toCheck
}

func (ms *memStore) deleteDeadSources(zid id.Zid, zi *zettelData) {
	// Must only be called if ms.mx is write-locked!
	for _, ref := range zi.dead {
		if drefs, ok := ms.dead[ref]; ok {
			drefs = remRef(drefs, zid)
			if len(drefs) > 0 {
				ms.dead[ref] = drefs
			} else {
				delete(ms.dead, ref)
			}
		}
	}
}

func (ms *memStore) deleteForwardBackward(zid id.Zid, zi *zettelData) id.Set {
	// Must only be called if ms.mx is write-locked!
	for _, ref := range zi.forward {
		if fzi, ok := ms.idx[ref]; ok {
			fzi.backward = remRef(fzi.backward, zid)
		}

	}
	var toCheck id.Set
	for _, ref := range zi.backward {
		if bzi, ok := ms.idx[ref]; ok {
			bzi.forward = remRef(bzi.forward, zid)
			toCheck = toCheck.Add(ref)
		}
	}
	return toCheck
}

func (ms *memStore) removeInverseMeta(zid id.Zid, key string, forward id.Slice) {
	// Must only be called if ms.mx is write-locked!
	for _, ref := range forward {
		bzi, ok := ms.idx[ref]
		if !ok || bzi.otherRefs == nil {
			continue
		}
		bmr, ok := bzi.otherRefs[key]
		if !ok {
			continue
		}
		bmr.backward = remRef(bmr.backward, zid)
		if len(bmr.backward) > 0 || len(bmr.forward) > 0 {
			bzi.otherRefs[key] = bmr
		} else {
			delete(bzi.otherRefs, key)
			if len(bzi.otherRefs) == 0 {
				bzi.otherRefs = nil
			}
		}
	}
}

func deleteStrings(msStringMap stringRefs, curStrings []string, zid id.Zid) {
	// Must only be called if ms.mx is write-locked!
	for _, word := range curStrings {
		refs, ok := msStringMap[word]
		if !ok {
			continue
		}
		refs2 := remRef(refs, zid)
		if len(refs2) == 0 {
			delete(msStringMap, word)
			continue
		}
		msStringMap[word] = refs2
















	}
}

func (ms *memStore) ReadStats(st *store.Stats) {
	ms.mx.RLock()
	st.Zettel = len(ms.idx)
	st.Words = uint64(len(ms.words))
	st.Urls = uint64(len(ms.urls))
	ms.mx.RUnlock()
	ms.mxStats.Lock()
	st.Updates = ms.updates
	ms.mxStats.Unlock()
}

func (ms *memStore) Dump(w io.Writer) {
	ms.mx.RLock()
	defer ms.mx.RUnlock()

	io.WriteString(w, "=== Dump\n")
	ms.dumpIndex(w)
	ms.dumpDead(w)
	dumpStringRefs(w, "Words", "", "", ms.words)
	dumpStringRefs(w, "URLs", "[[", "]]", ms.urls)
}

func (ms *memStore) dumpIndex(w io.Writer) {
	if len(ms.idx) == 0 {
		return
	}
	io.WriteString(w, "==== Zettel Index\n")
	zids := make(id.Slice, 0, len(ms.idx))
	for id := range ms.idx {
		zids = append(zids, id)
	}
	zids.Sort()
	for _, id := range zids {
		fmt.Fprintln(w, "=====", id)
		zi := ms.idx[id]
		if len(zi.dead) > 0 {
			fmt.Fprintln(w, "* Dead:", zi.dead)
		}
		dumpZids(w, "* Forward:", zi.forward)
		dumpZids(w, "* Backward:", zi.backward)

		otherRefs := make([]string, 0, len(zi.otherRefs))
		for k := range zi.otherRefs {
			otherRefs = append(otherRefs, k)
		}
		slices.Sort(otherRefs)
		for _, k := range otherRefs {
			fmt.Fprintln(w, "* Meta", k)
			dumpZids(w, "** Forward:", zi.otherRefs[k].forward)
			dumpZids(w, "** Backward:", zi.otherRefs[k].backward)
		}
		dumpStrings(w, "* Words", "", "", zi.words)
		dumpStrings(w, "* URLs", "[[", "]]", zi.urls)
	}
}

func (ms *memStore) dumpDead(w io.Writer) {
	if len(ms.dead) == 0 {
		return
	}
	fmt.Fprintf(w, "==== Dead References\n")
	zids := make(id.Slice, 0, len(ms.dead))
	for id := range ms.dead {
		zids = append(zids, id)
	}
	zids.Sort()
	for _, id := range zids {
		fmt.Fprintln(w, ";", id)
		fmt.Fprintln(w, ":", ms.dead[id])
	}
}

func dumpZids(w io.Writer, prefix string, zids id.Slice) {
	if len(zids) > 0 {
		io.WriteString(w, prefix)
		for _, zid := range zids {
			io.WriteString(w, " ")
			w.Write(zid.Bytes())
		}
		fmt.Fprintln(w)
	}
}

func dumpStrings(w io.Writer, title, preString, postString string, slice []string) {
	if len(slice) > 0 {
		sl := make([]string, len(slice))
		copy(sl, slice)
		slices.Sort(sl)
		fmt.Fprintln(w, title)
		for _, s := range sl {

//-----------------------------------------------------------------------------
// Copyright (c) 2021-present Detlef Stern
//
// This file is part of Zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//
// SPDX-License-Identifier: EUPL-1.2
// SPDX-FileCopyrightText: 2021-present Detlef Stern
//-----------------------------------------------------------------------------

package mapstore

import (
	"slices"

	"zettelstore.de/z/zettel/id"
)

func refsDiff(refsN, refsO id.Slice) (newRefs, remRefs id.Slice) {
	npos, opos := 0, 0
	for npos < len(refsN) && opos < len(refsO) {
		rn, ro := refsN[npos], refsO[opos]
		if rn == ro {
			npos++
			opos++
			continue
		}
		if rn < ro {
			newRefs = append(newRefs, rn)
			npos++
			continue
		}
		remRefs = append(remRefs, ro)
		opos++
	}
	if npos < len(refsN) {
		newRefs = append(newRefs, refsN[npos:]...)
	}
	if opos < len(refsO) {
		remRefs = append(remRefs, refsO[opos:]...)
	}
	return newRefs, remRefs
}

func addRef(refs id.Slice, ref id.Zid) id.Slice {
	hi := len(refs)
	for lo := 0; lo < hi; {
		m := lo + (hi-lo)/2
		if r := refs[m]; r == ref {
			return refs
		} else if r < ref {
			lo = m + 1
		} else {
			hi = m
		}
	}
	refs = slices.Insert(refs, hi, ref)
	return refs
}

func remRefs(refs, rem id.Slice) id.Slice {
	if len(refs) == 0 || len(rem) == 0 {
		return refs
	}
	result := make(id.Slice, 0, len(refs))
	rpos, dpos := 0, 0
	for rpos < len(refs) && dpos < len(rem) {
		rr, dr := refs[rpos], rem[dpos]
		if rr < dr {
			result = append(result, rr)
			rpos++
			continue
		}
		if dr < rr {
			dpos++
			continue
		}
		rpos++
		dpos++
	}
	if rpos < len(refs) {
		result = append(result, refs[rpos:]...)
	}
	return result
}

func remRef(refs id.Slice, ref id.Zid) id.Slice {
	hi := len(refs)
	for lo := 0; lo < hi; {
		m := lo + (hi-lo)/2
		if r := refs[m]; r == ref {
			copy(refs[m:], refs[m+1:])
			refs = refs[:len(refs)-1]
			return refs
		} else if r < ref {
			lo = m + 1
		} else {
			hi = m
		}
	}
	return refs
}

//-----------------------------------------------------------------------------
// Copyright (c) 2021-present Detlef Stern
//
// This file is part of Zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//
// SPDX-License-Identifier: EUPL-1.2
// SPDX-FileCopyrightText: 2021-present Detlef Stern
//-----------------------------------------------------------------------------

package mapstore

import (
	"testing"

	"zettelstore.de/z/zettel/id"
)

func assertRefs(t *testing.T, i int, got, exp id.Slice) {
	t.Helper()
	if got == nil && exp != nil {
		t.Errorf("%d: got nil, but expected %v", i, exp)
		return
	}
	if got != nil && exp == nil {
		t.Errorf("%d: expected nil, but got %v", i, got)
		return
	}
	if len(got) != len(exp) {
		t.Errorf("%d: expected len(%v)==%d, but got len(%v)==%d", i, exp, len(exp), got, len(got))
		return
	}
	for p, n := range exp {
		if got := got[p]; got != id.Zid(n) {
			t.Errorf("%d: pos %d: expected %d, but got %d", i, p, n, got)
		}
	}
}

func TestRefsDiff(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		in1, in2   id.Slice
		exp1, exp2 id.Slice
	}{
		{nil, nil, nil, nil},
		{id.Slice{1}, nil, id.Slice{1}, nil},
		{nil, id.Slice{1}, nil, id.Slice{1}},
		{id.Slice{1}, id.Slice{1}, nil, nil},
		{id.Slice{1, 2}, id.Slice{1}, id.Slice{2}, nil},
		{id.Slice{1, 2}, id.Slice{1, 3}, id.Slice{2}, id.Slice{3}},
		{id.Slice{1, 4}, id.Slice{1, 3}, id.Slice{4}, id.Slice{3}},
	}
	for i, tc := range testcases {
		got1, got2 := refsDiff(tc.in1, tc.in2)
		assertRefs(t, i, got1, tc.exp1)
		assertRefs(t, i, got2, tc.exp2)
	}
}

func TestAddRef(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		ref id.Slice
		zid uint
		exp id.Slice
	}{
		{nil, 5, id.Slice{5}},
		{id.Slice{1}, 5, id.Slice{1, 5}},
		{id.Slice{10}, 5, id.Slice{5, 10}},
		{id.Slice{5}, 5, id.Slice{5}},
		{id.Slice{1, 10}, 5, id.Slice{1, 5, 10}},
		{id.Slice{1, 5, 10}, 5, id.Slice{1, 5, 10}},
	}
	for i, tc := range testcases {
		got := addRef(tc.ref, id.Zid(tc.zid))
		assertRefs(t, i, got, tc.exp)
	}
}

func TestRemRefs(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		in1, in2 id.Slice
		exp      id.Slice
	}{
		{nil, nil, nil},
		{nil, id.Slice{}, nil},
		{id.Slice{}, nil, id.Slice{}},
		{id.Slice{}, id.Slice{}, id.Slice{}},
		{id.Slice{1}, id.Slice{5}, id.Slice{1}},
		{id.Slice{10}, id.Slice{5}, id.Slice{10}},
		{id.Slice{1, 5}, id.Slice{5}, id.Slice{1}},
		{id.Slice{5, 10}, id.Slice{5}, id.Slice{10}},
		{id.Slice{1, 10}, id.Slice{5}, id.Slice{1, 10}},
		{id.Slice{1}, id.Slice{2, 5}, id.Slice{1}},
		{id.Slice{10}, id.Slice{2, 5}, id.Slice{10}},
		{id.Slice{1, 5}, id.Slice{2, 5}, id.Slice{1}},
		{id.Slice{5, 10}, id.Slice{2, 5}, id.Slice{10}},
		{id.Slice{1, 2, 5}, id.Slice{2, 5}, id.Slice{1}},
		{id.Slice{2, 5, 10}, id.Slice{2, 5}, id.Slice{10}},
		{id.Slice{1, 10}, id.Slice{2, 5}, id.Slice{1, 10}},
		{id.Slice{1}, id.Slice{5, 9}, id.Slice{1}},
		{id.Slice{10}, id.Slice{5, 9}, id.Slice{10}},
		{id.Slice{1, 5}, id.Slice{5, 9}, id.Slice{1}},
		{id.Slice{5, 10}, id.Slice{5, 9}, id.Slice{10}},
		{id.Slice{1, 5, 9}, id.Slice{5, 9}, id.Slice{1}},
		{id.Slice{5, 9, 10}, id.Slice{5, 9}, id.Slice{10}},
		{id.Slice{1, 10}, id.Slice{5, 9}, id.Slice{1, 10}},
	}
	for i, tc := range testcases {
		got := remRefs(tc.in1, tc.in2)
		assertRefs(t, i, got, tc.exp)
	}
}

func TestRemRef(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		ref id.Slice
		zid uint
		exp id.Slice
	}{
		{nil, 5, nil},
		{id.Slice{}, 5, id.Slice{}},
		{id.Slice{5}, 5, id.Slice{}},
		{id.Slice{1}, 5, id.Slice{1}},
		{id.Slice{10}, 5, id.Slice{10}},
		{id.Slice{1, 5}, 5, id.Slice{1}},
		{id.Slice{5, 10}, 5, id.Slice{10}},
		{id.Slice{1, 5, 10}, 5, id.Slice{1, 10}},
	}
	for i, tc := range testcases {
		got := remRef(tc.ref, id.Zid(tc.zid))
		assertRefs(t, i, got, tc.exp)
	}
}

	GetMeta(context.Context, id.Zid) (*meta.Meta, error)

	// Entrich metadata with data from store.
	Enrich(ctx context.Context, m *meta.Meta)

	// UpdateReferences for a specific zettel.
	// Returns set of zettel identifier that must also be checked for changes.
	UpdateReferences(context.Context, *ZettelIndex) id.Set

	// RenameZettel changes all references of current zettel identifier to new
	// zettel identifier.
	RenameZettel(_ context.Context, curZid, newZid id.Zid) id.Set

	// DeleteZettel removes index data for given zettel.
	// Returns set of zettel identifier that must also be checked for changes.
	DeleteZettel(context.Context, id.Zid) id.Set




	// ReadStats populates st with store statistics.
	ReadStats(st *Stats)

	// Dump the content to a Writer.
	Dump(io.Writer)
}

import (
	"zettelstore.de/z/zettel/id"
	"zettelstore.de/z/zettel/meta"
)

// ZettelIndex contains all index data of a zettel.
type ZettelIndex struct {
	Zid         id.Zid            // zid of the indexed zettel
	meta        *meta.Meta        // full metadata
	backrefs    id.Set            // set of back references
	inverseRefs map[string]id.Set // references of inverse keys
	deadrefs    id.Set            // set of dead references
	words       WordSet
	urls        WordSet
}

// NewZettelIndex creates a new zettel index.
func NewZettelIndex(m *meta.Meta) *ZettelIndex {
	return &ZettelIndex{
		Zid:         m.Zid,
		meta:        m,
		backrefs:    id.NewSet(),
		inverseRefs: make(map[string]id.Set),
		deadrefs:    id.NewSet(),
	}
}

// AddBackRef adds a reference to a zettel where the current zettel links to
// without any more information.
func (zi *ZettelIndex) AddBackRef(zid id.Zid) {
	zi.backrefs.Add(zid)
}

// AddInverseRef adds a named reference to a zettel. On that zettel, the given
// metadata key should point back to the current zettel.
func (zi *ZettelIndex) AddInverseRef(key string, zid id.Zid) {
	if zids, ok := zi.inverseRefs[key]; ok {
		zids.Add(zid)
		return

// SetWords sets the words to the given value.
func (zi *ZettelIndex) SetWords(words WordSet) { zi.words = words }

// SetUrls sets the words to the given value.
func (zi *ZettelIndex) SetUrls(urls WordSet) { zi.urls = urls }

// GetDeadRefs returns all dead references as a sorted list.
func (zi *ZettelIndex) GetDeadRefs() id.Slice { return zi.deadrefs.Sorted() }

// GetMeta return just the raw metadata.
func (zi *ZettelIndex) GetMeta() *meta.Meta { return zi.meta }

// GetBackRefs returns all back references as a sorted list.
func (zi *ZettelIndex) GetBackRefs() id.Slice { return zi.backrefs.Sorted() }

// GetInverseRefs returns all inverse meta references as a map of strings to a sorted list of references
func (zi *ZettelIndex) GetInverseRefs() map[string]id.Slice {
	if len(zi.inverseRefs) == 0 {
		return nil
	}
	result := make(map[string]id.Slice, len(zi.inverseRefs))
	for key, refs := range zi.inverseRefs {
		result[key] = refs.Sorted()
	}
	return result
}

// GetWords returns a reference to the set of words. It must not be modified.
func (zi *ZettelIndex) GetWords() WordSet { return zi.words }

// GetUrls returns a reference to the set of URLs. It must not be modified.
func (zi *ZettelIndex) GetUrls() WordSet { return zi.urls }

id: 00001012053500
title: API: Retrieve evaluated metadata and content of an existing zettel in various encodings
role: manual
tags: #api #manual #zettelstore
syntax: zmk
created: 20210726174524
modified: 20230807170112

The [[endpoint|00001012920000]] to work with evaluated metadata and content of a specific zettel is ''/z/{ID}'', where ''{ID}'' is a placeholder for the [[zettel identifier|00001006050000]].

For example, to retrieve some evaluated data about this zettel you are currently viewing in [[Sz encoding|00001012920516]], just send a HTTP GET request to the endpoint ''/z/00001012053500''[^If [[authentication is enabled|00001010040100]], you must include the a valid [[access token|00001012050200]] in the ''Authorization'' header] with the query parameter ''enc=sz''.
If successful, the output is a symbolic expression value:
```sh
# curl 'http://127.0.0.1:23123/z/00001012053500?enc=sz'
((PARA (TEXT "The") (SPACE) (LINK-ZETTEL () "00001012920000" (TEXT "endpoint")) (SPACE) (TEXT "to") (SPACE) (TEXT "work") (SPACE) (TEXT "with") (SPACE) (TEXT "evaluated") (SPACE) (TEXT "metadata") (SPACE) (TEXT "and") (SPACE) (TEXT "content") (SPACE) (TEXT "of") (SPACE) (TEXT "a") (SPACE) (TEXT "specific") (SPACE) (TEXT "zettel") (SPACE) (TEXT "is") (SPACE) (LITERAL-INPUT () "/z/{ID}") (TEXT ",") (SPACE) (TEXT "where") (SPACE) (LITERAL-INPUT () "{ID}") ...
```

To select another encoding, you must provide the query parameter ''enc=ENCODING''.
Others are ""[[html|00001012920510]]"", ""[[text|00001012920519]]"", and some [[more|00001012920500]].
In addition, you may provide a query parameter ''part=PART'' to select the relevant [[part|00001012920800]] of a zettel.
```sh
# curl 'http://127.0.0.1:23123/z/00001012053500?enc=html&part=zettel'

id: 00001012053600
title: API: Retrieve parsed metadata and content of an existing zettel in various encodings
role: manual
tags: #api #manual #zettelstore
syntax: zmk
created: 20210126175322
modified: 20230807170019

The [[endpoint|00001012920000]] to work with parsed metadata and content of a specific zettel is ''/z/{ID}'', where ''{ID}'' is a placeholder for the [[zettel identifier|00001006050000]].

A __parsed__ zettel is basically an [[unevaluated|00001012053500]] zettel: the zettel is read and analyzed, but its content is not __evaluated__.
By using this endpoint, you are able to retrieve the structure of a zettel before it is evaluated.

For example, to retrieve some data about this zettel you are currently viewing, just send a HTTP GET request to the endpoint ''/z/00001012053600''[^If [[authentication is enabled|00001010040100]], you must include the a valid [[access token|00001012050200]] in the ''Authorization'' header] with the query parameter ''parseonly'' (and other appropriate query parameter).
For example:
```sh
# curl 'http://127.0.0.1:23123/z/00001012053600?enc=sz&parseonly'
((PARA (TEXT "The") (SPACE) (LINK-ZETTEL () "00001012920000" (TEXT "endpoint")) (SPACE) (TEXT "to") (SPACE) (TEXT "work") (SPACE) (TEXT "with") (SPACE) ...
```

Similar to [[retrieving an encoded zettel|00001012053500]], you can specify an [[encoding|00001012920500]] and state which [[part|00001012920800]] of a zettel you are interested in.
The same default values applies to this endpoint.

=== HTTP Status codes
; ''200''

id: 00001012931600
title: Encoding of Sz Inline Elements
role: manual
tags: #api #manual #reference #zettelstore
syntax: zmk
created: 20230403161845
modified: 20240122122448

=== ''TEXT''
:::syntax
__Text__ **=** ''(TEXT'' String '')''.
:::
Specifies the string as some text content, typically a word.

=== ''SPACE''
:::syntax
__Space__ **=** ''(SPACE'' **[** String **]** '')''.
:::
Specifies some space, typically white space.
If the string is not given it is assumed to be ''" "'' (one space character).
Otherwise it contains the space characters.

=== ''SOFT''
:::syntax
__Soft__ **=** ''(SOFT)''.
:::
Denotes a soft line break.
It is typically translated into a space character, but signals the point in the textual content, where a line break occurred.

=== ''HARD''
:::syntax
__Hard__ **=** ''(HARD)''.
:::
Specifies a hard line break, i.e. the user wants to have a line break here.

	},
	{
		descr: "Simple text: Hello, world",
		zmk:   "Hello, world",
		expect: expectMap{
			encoderHTML:  "<p>Hello, world</p>",
			encoderMD:    "Hello, world",
			encoderSz:    `(BLOCK (PARA (TEXT "Hello,") (SPACE) (TEXT "world")))`,
			encoderSHTML: `((p "Hello," " " "world"))`,
			encoderText:  "Hello, world",
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Simple block comment",
		zmk:   "%%%\nNo\nrender\n%%%",

	},
	{
		descr: "Simple Heading",
		zmk:   `=== Top Job`,
		expect: expectMap{
			encoderHTML:  "<h2 id=\"top-job\">Top Job</h2>",
			encoderMD:    "# Top Job",
			encoderSz:    `(BLOCK (HEADING 1 () "top-job" "top-job" (TEXT "Top") (SPACE) (TEXT "Job")))`,
			encoderSHTML: `((h2 (@ (id . "top-job")) "Top" " " "Job"))`,
			encoderText:  `Top Job`,
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Simple List",
		zmk:   "* A\n* B\n* C",

	},
	{
		descr: "Simple Quote Block",
		zmk:   "<<<\nToBeOrNotToBe\n<<< Romeo Julia",
		expect: expectMap{
			encoderHTML:  "<blockquote><p>ToBeOrNotToBe</p><cite>Romeo Julia</cite></blockquote>",
			encoderMD:    "> ToBeOrNotToBe",
			encoderSz:    `(BLOCK (REGION-QUOTE () ((PARA (TEXT "ToBeOrNotToBe"))) (TEXT "Romeo") (SPACE) (TEXT "Julia")))`,
			encoderSHTML: `((blockquote (p "ToBeOrNotToBe") (cite "Romeo" " " "Julia")))`,
			encoderText:  "ToBeOrNotToBe\nRomeo Julia",
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Quote Block with multiple paragraphs",
		zmk:   "<<<\nToBeOr\n\nNotToBe\n<<< Romeo",

Paragraph

    Spacy  Para
""" Author`,
		expect: expectMap{
			encoderHTML:  "<div><p>A\u00a0line<br>\u00a0\u00a0another\u00a0line<br>Back</p><p>Paragraph</p><p>\u00a0\u00a0\u00a0\u00a0Spacy\u00a0\u00a0Para</p><cite>Author</cite></div>",
			encoderMD:    "",
			encoderSz:    "(BLOCK (REGION-VERSE () ((PARA (TEXT \"A\") (SPACE \"\u00a0\") (TEXT \"line\") (HARD) (SPACE \"\u00a0\u00a0\") (TEXT \"another\") (SPACE \"\u00a0\") (TEXT \"line\") (HARD) (TEXT \"Back\")) (PARA (TEXT \"Paragraph\")) (PARA (SPACE \"\u00a0\u00a0\u00a0\u00a0\") (TEXT \"Spacy\") (SPACE \"\u00a0\u00a0\") (TEXT \"Para\"))) (TEXT \"Author\")))",
			encoderSHTML: "((div (p \"A\" \"\u00a0\" \"line\" (br) \"\u00a0\u00a0\" \"another\" \"\u00a0\" \"line\" (br) \"Back\") (p \"Paragraph\") (p \"\u00a0\u00a0\u00a0\u00a0\" \"Spacy\" \"\u00a0\u00a0\" \"Para\") (cite \"Author\")))",
			encoderText:  "A line\n another line\nBack\nParagraph\n Spacy Para\nAuthor",
			encoderZmk:   "\"\"\"\nA\u00a0line\\\n\u00a0\u00a0another\u00a0line\\\nBack\nParagraph\n\u00a0\u00a0\u00a0\u00a0Spacy\u00a0\u00a0Para\n\"\"\" Author",
		},
	},
	{
		descr: "Span Block",
		zmk: `:::
A simple
   span
and much more
:::`,
		expect: expectMap{
			encoderHTML:  "<div><p>A simple  span and much more</p></div>",
			encoderMD:    "",
			encoderSz:    `(BLOCK (REGION-BLOCK () ((PARA (TEXT "A") (SPACE) (TEXT "simple") (SOFT) (SPACE) (TEXT "span") (SOFT) (TEXT "and") (SPACE) (TEXT "much") (SPACE) (TEXT "more")))))`,
			encoderSHTML: `((div (p "A" " " "simple" " " " " "span" " " "and" " " "much" " " "more")))`,
			encoderText:  `A simple  span and much more`,
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Simple Verbatim Code",
		zmk:   "```\nHello\nWorld\n```",

	},
	{
		descr: "Simple Description List",
		zmk:   "; Zettel\n: Paper\n: Note\n; Zettelkasten\n: Slip box",
		expect: expectMap{
			encoderHTML:  "<dl><dt>Zettel</dt><dd><p>Paper</p></dd><dd><p>Note</p></dd><dt>Zettelkasten</dt><dd><p>Slip box</p></dd></dl>",
			encoderMD:    "",
			encoderSz:    `(BLOCK (DESCRIPTION ((TEXT "Zettel")) (BLOCK (BLOCK (PARA (TEXT "Paper"))) (BLOCK (PARA (TEXT "Note")))) ((TEXT "Zettelkasten")) (BLOCK (BLOCK (PARA (TEXT "Slip") (SPACE) (TEXT "box"))))))`,
			encoderSHTML: `((dl (dt "Zettel") (dd (p "Paper")) (dd (p "Note")) (dt "Zettelkasten") (dd (p "Slip" " " "box"))))`,
			encoderText:  "Zettel\nPaper\nNote\nZettelkasten\nSlip box",
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Description List with paragraphs as item",
		zmk:   "; Zettel\n: Paper\n\n  Note\n; Zettelkasten\n: Slip box",
		expect: expectMap{
			encoderHTML:  "<dl><dt>Zettel</dt><dd><p>Paper</p><p>Note</p></dd><dt>Zettelkasten</dt><dd><p>Slip box</p></dd></dl>",
			encoderMD:    "",
			encoderSz:    `(BLOCK (DESCRIPTION ((TEXT "Zettel")) (BLOCK (BLOCK (PARA (TEXT "Paper")) (PARA (TEXT "Note")))) ((TEXT "Zettelkasten")) (BLOCK (BLOCK (PARA (TEXT "Slip") (SPACE) (TEXT "box"))))))`,
			encoderSHTML: `((dl (dt "Zettel") (dd (p "Paper") (p "Note")) (dt "Zettelkasten") (dd (p "Slip" " " "box"))))`,
			encoderText:  "Zettel\nPaper\nNote\nZettelkasten\nSlip box",
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Description List with keys, but no descriptions",
		zmk:   "; K1\n: D11\n: D12\n; K2\n; K3\n: D31",

	},
	{
		descr: "Simple text: Hello, world (inline)",
		zmk:   `Hello, world`,
		expect: expectMap{
			encoderHTML:  "Hello, world",
			encoderMD:    "Hello, world",
			encoderSz:    `(INLINE (TEXT "Hello,") (SPACE) (TEXT "world"))`,
			encoderSHTML: `("Hello," " " "world")`,
			encoderText:  "Hello, world",
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Soft Break",
		zmk:   "soft\nbreak",

	},
	{
		descr: "Nested quotes (default)",
		zmk:   `""say: ::""yes, ::""or?""::""::""`,
		expect: expectMap{
			encoderHTML:  `&ldquo;say: <span>&lsquo;yes, <span>&ldquo;or?&rdquo;</span>&rsquo;</span>&rdquo;`,
			encoderMD:    "<q>say: <q>yes, <q>or?</q></q></q>",
			encoderSz:    `(INLINE (FORMAT-QUOTE () (TEXT "say:") (SPACE) (FORMAT-SPAN () (FORMAT-QUOTE () (TEXT "yes,") (SPACE) (FORMAT-SPAN () (FORMAT-QUOTE () (TEXT "or?")))))))`,
			encoderSHTML: `((@L (@H "&ldquo;") "say:" " " (span (@L (@H "&lsquo;") "yes," " " (span (@L (@H "&ldquo;") "or?" (@H "&rdquo;"))) (@H "&rsquo;"))) (@H "&rdquo;")))`,
			encoderText:  `say: yes, or?`,
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Two quotes",
		zmk:   `""yes"" or ""no""`,
		expect: expectMap{
			encoderHTML:  `&ldquo;yes&rdquo; or &ldquo;no&rdquo;`,
			encoderMD:    "<q>yes</q> or <q>no</q>",
			encoderSz:    `(INLINE (FORMAT-QUOTE () (TEXT "yes")) (SPACE) (TEXT "or") (SPACE) (FORMAT-QUOTE () (TEXT "no")))`,
			encoderSHTML: `((@L (@H "&ldquo;") "yes" (@H "&rdquo;")) " " "or" " " (@L (@H "&ldquo;") "no" (@H "&rdquo;")))`,
			encoderText:  `yes or no`,
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Mark formatting",
		zmk:   `##marked##`,

	},
	{
		descr: "HTML in Code formatting",
		zmk:   "``<script `` abc",
		expect: expectMap{
			encoderHTML:  "<code>&lt;script </code> abc",
			encoderMD:    "`<script ` abc",
			encoderSz:    `(INLINE (LITERAL-CODE () "<script ") (SPACE) (TEXT "abc"))`,
			encoderSHTML: `((code "<script ") " " "abc")`,
			encoderText:  `<script  abc`,
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Input formatting",
		zmk:   `''input''`,

		},
	}, {
		descr: "No comment",
		zmk:   `% comment`,
		expect: expectMap{
			encoderHTML:  `% comment`,
			encoderMD:    "% comment",
			encoderSz:    `(INLINE (TEXT "%") (SPACE) (TEXT "comment"))`,
			encoderSHTML: `("%" " " "comment")`,
			encoderText:  `% comment`,
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Line comment (nogen HTML)",
		zmk:   `%% line comment`,

	},
	{
		descr: "Mark with text",
		zmk:   `[!mark|with text]`,
		expect: expectMap{
			encoderHTML:  `<a id="mark">with text</a>`,
			encoderMD:    "with text",
			encoderSz:    `(INLINE (MARK "mark" "mark" "mark" (TEXT "with") (SPACE) (TEXT "text")))`,
			encoderSHTML: `((a (@ (id . "mark")) "with" " " "text"))`,
			encoderText:  `with text`,
			encoderZmk:   useZmk,
		},
	},
	{
		descr: "Invalid Link",
		zmk:   `[[link|00000000000000]]`,

		v.visitNestedList(n)
	case *ast.DescriptionListNode:
		return nil // Should write no content
	case *ast.TableNode:
		return nil // Should write no content
	case *ast.TextNode:
		v.b.WriteString(n.Text)
	case *ast.SpaceNode:
		v.b.WriteString(n.Lexeme)
	case *ast.BreakNode:
		v.visitBreak(n)
	case *ast.LinkNode:
		v.visitLink(n)
	case *ast.EmbedRefNode:
		v.visitEmbedRef(n)
	case *ast.FootnoteNode:

		return t.getTable(n)
	case *ast.TranscludeNode:
		return sx.MakeList(sz.SymTransclude, getAttributes(n.Attrs), getReference(n.Ref))
	case *ast.BLOBNode:
		return t.getBLOB(n)
	case *ast.TextNode:
		return sx.MakeList(sz.SymText, sx.MakeString(n.Text))
	case *ast.SpaceNode:
		if t.inVerse {
			return sx.MakeList(sz.SymSpace, sx.MakeString(n.Lexeme))
		}
		return sx.MakeList(sz.SymSpace)
	case *ast.BreakNode:
		if n.Hard {
			return sx.MakeList(sz.SymHard)
		}
		return sx.MakeList(sz.SymSoft)
	case *ast.LinkNode:
		return t.getLink(n)

// Package textenc encodes the abstract syntax tree into its text.
package textenc

import (
	"io"

	"t73f.de/r/zsc/api"

	"zettelstore.de/z/ast"
	"zettelstore.de/z/encoder"
	"zettelstore.de/z/zettel/meta"
)

func init() {
	encoder.Register(api.EncoderText, func(*encoder.CreateParameter) encoder.Encoder { return Create() })

		return nil
	case *ast.TableNode:
		v.visitTable(n)
		return nil
	case *ast.TranscludeNode, *ast.BLOBNode:
		return nil
	case *ast.TextNode:
		v.b.WriteString(n.Text)
		return nil
	case *ast.SpaceNode:
		v.b.WriteByte(' ')
		return nil
	case *ast.BreakNode:
		if n.Hard {
			v.b.WriteByte('\n')
		} else {
			v.b.WriteByte(' ')
		}

func (v *visitor) visitInlineSlice(is *ast.InlineSlice) {
	for i, in := range *is {
		v.inlinePos = i
		ast.Walk(v, in)
	}
	v.inlinePos = 0
}
















func (v *visitor) writePosChar(pos int, ch byte) {
	if pos > 0 {
		v.b.WriteByte(ch)
	}
}

	case *ast.TranscludeNode:
		v.b.WriteStrings("{{{", n.Ref.String(), "}}}")
		v.visitAttributes(n.Attrs)
	case *ast.BLOBNode:
		v.visitBLOB(n)
	case *ast.TextNode:
		v.visitText(n)
	case *ast.SpaceNode:
		v.b.WriteString(n.Lexeme)
	case *ast.BreakNode:
		v.visitBreak(n)
	case *ast.LinkNode:
		v.visitLink(n)
	case *ast.EmbedRefNode:
		v.visitEmbedRef(n)
	case *ast.EmbedBLOBNode:

		ast.Walk(fs, bn)
	}
}

func (fs *fragmentSearcher) visitInlineSlice(is *ast.InlineSlice) {
	for i, in := range *is {
		if mn, ok := in.(*ast.MarkNode); ok && mn.Fragment == fs.fragment {
			ris := skipSpaceNodes((*is)[i+1:])
			if len(mn.Inlines) > 0 {
				fs.result = append(ast.InlineSlice{}, mn.Inlines...)
				fs.result = append(fs.result, &ast.SpaceNode{Lexeme: " "})
				fs.result = append(fs.result, ris...)
			} else {
				fs.result = ris
			}
			return
		}
		ast.Walk(fs, in)
	}
}

func skipSpaceNodes(ins ast.InlineSlice) ast.InlineSlice {
	for i, in := range ins {
		switch in.(type) {
		case *ast.SpaceNode:
		case *ast.BreakNode:
		default:
			return ins[i:]
		}
	}
	return nil
}

	ccs = ap.limitTags(ccs)
	countMap := ap.calcFontSizes(ccs)

	para := make(ast.InlineSlice, 0, len(ccs))
	ccs.SortByName()
	for i, cat := range ccs {
		if i > 0 {
			para = append(para, &ast.SpaceNode{Lexeme: " "})
		}
		buf.WriteString(cat.Name)
		para = append(para,
			&ast.LinkNode{
				Attrs: countMap[cat.Count],
				Ref:   ast.ParseReference(buf.String()),
				Inlines: ast.InlineSlice{

		items = append(items, ast.ItemSlice{ast.CreateParaNode(
			&ast.LinkNode{
				Attrs:   nil,
				Ref:     ast.ParseReference(q1),
				Inlines: ast.InlineSlice{&ast.TextNode{Text: cat.Name}},
			},
			&ast.SpaceNode{Lexeme: " "},
			&ast.TextNode{Text: "(" + strconv.Itoa(cat.Count) + ", "},
			&ast.LinkNode{
				Attrs:   nil,
				Ref:     ast.ParseReference(q2),
				Inlines: ast.InlineSlice{&ast.TextNode{Text: "values"}},
			},
			&ast.TextNode{Text: ")"},

		sliceData = []string{value}
	}
	makeLink := dt == meta.TypeID || dt == meta.TypeIDSet

	result := make(ast.InlineSlice, 0, 2*len(sliceData)-1)
	for i, val := range sliceData {
		if i > 0 {
			result = append(result, &ast.SpaceNode{Lexeme: " "})
		}
		tn := &ast.TextNode{Text: val}
		if makeLink {
			result = append(result, &ast.LinkNode{
				Ref:     ast.ParseReference(val),
				Inlines: ast.InlineSlice{tn},
			})

module zettelstore.de/z

go 1.22

require (
	github.com/fsnotify/fsnotify v1.7.0
	github.com/yuin/goldmark v1.7.3
	golang.org/x/crypto v0.24.0
	golang.org/x/term v0.21.0
	golang.org/x/text v0.16.0
	t73f.de/r/sx v0.0.0-20240513163553-ec4fcc6539ca
	t73f.de/r/sxwebs v0.0.0-20240613142113-66fc5a284245
	t73f.de/r/zsc v0.0.0-20240531154807-955147c904a3
)

require (
	golang.org/x/sys v0.21.0 // indirect
	t73f.de/r/webs v0.0.0-20240607145146-ac6799458514 // indirect
)

golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
t73f.de/r/sx v0.0.0-20240513163553-ec4fcc6539ca h1:vvDqiuUfBLf+t/gpiSyqIFAdvZ7FLigOH38bqMY+v8k=
t73f.de/r/sx v0.0.0-20240513163553-ec4fcc6539ca/go.mod h1:G9pD1j2R6y9ZkPBb81mSnmwaAvTOg7r6jKp/OF7WeFA=
t73f.de/r/sxwebs v0.0.0-20240613142113-66fc5a284245 h1:raE7KUgoGsp2DzXOko9dDXEsSJ/VvoXCDYeICx7i6uo=
t73f.de/r/sxwebs v0.0.0-20240613142113-66fc5a284245/go.mod h1:ErPBVUyE2fOktL/8M7lp/PR93wP/o9RawMajB1uSqj8=
t73f.de/r/webs v0.0.0-20240607145146-ac6799458514 h1:sXTOGkpdB0FT3cphsmj65ZAiX/SntlC0GzLif1EipMY=
t73f.de/r/webs v0.0.0-20240607145146-ac6799458514/go.mod h1:UGAAtul0TK5ACeZ6zTS3SX6GqwMFXxlUpHiV8oqNq5w=
t73f.de/r/zsc v0.0.0-20240531154807-955147c904a3 h1:iY1X8gmZ/YNsFysPQAGVpTOPknI7hxbGhWKxdDgCvr8=
t73f.de/r/zsc v0.0.0-20240531154807-955147c904a3/go.mod h1:YdsjqbI1th0bJoMclgbNycJUVr8ovAV02ZgA91uOhkU=

			return n
		}
	}
	return defVal
}

func canvasErrMsg(err error) ast.InlineSlice {
	return ast.CreateInlineSliceFromWords("Error:", err.Error())
}

func noSVGErrMsg() ast.InlineSlice {
	return ast.CreateInlineSliceFromWords("NO", "IMAGE")
}

		return p.acceptRawHTML(n)
	}
	panic(fmt.Sprintf("Unhandled inline node %v", node.Kind()))
}

func (p *mdP) acceptText(node *gmAst.Text) ast.InlineSlice {
	segment := node.Segment




	if node.IsRaw() {
		return splitText(string(segment.Value(p.source)))
	}
	ins := splitText(string(segment.Value(p.source)))
	result := make(ast.InlineSlice, 0, len(ins)+1)
	for _, in := range ins {
		if tn, ok := in.(*ast.TextNode); ok {
			tn.Text = cleanText([]byte(tn.Text), true)
		}
		result = append(result, in)
	}
	if node.HardLineBreak() {
		result = append(result, &ast.BreakNode{Hard: true})
	} else if node.SoftLineBreak() {
		result = append(result, &ast.BreakNode{Hard: false})
	}
	return result
}

// splitText transform the text into a sequence of TextNode and SpaceNode
func splitText(text string) ast.InlineSlice {
	if text == "" {
		return nil
	}
	result := make(ast.InlineSlice, 0, 1)

	state := 0 // 0=unknown,1=non-spaces,2=spaces
	lastPos := 0
	for pos, ch := range text {
		if input.IsSpace(ch) {
			if state == 1 {
				result = append(result, &ast.TextNode{Text: text[lastPos:pos]})
				lastPos = pos
			}
			state = 2
		} else {
			if state == 2 {
				result = append(result, &ast.SpaceNode{Lexeme: text[lastPos:pos]})
				lastPos = pos
			}
			state = 1
		}
	}
	switch state {
	case 1:
		result = append(result, &ast.TextNode{Text: text[lastPos:]})
	case 2:
		result = append(result, &ast.SpaceNode{Lexeme: text[lastPos:]})
	default:
		panic(fmt.Sprintf("Unexpected state %v", state))
	}
	return result
}

var ignoreAfterBS = map[byte]struct{}{
	'!': {}, '"': {}, '#': {}, '$': {}, '%': {}, '&': {}, '\'': {}, '(': {},
	')': {}, '*': {}, '+': {}, ',': {}, '-': {}, '.': {}, '/': {}, ':': {},
	';': {}, '<': {}, '=': {}, '>': {}, '?': {}, '@': {}, '[': {}, '\\': {},

//-----------------------------------------------------------------------------
// Copyright (c) 2020-present Detlef Stern
//
// This file is part of Zettelstore.
//
// Zettelstore is licensed under the latest version of the EUPL (European Union
// Public License). Please see file LICENSE.txt for your rights and obligations
// under this license.
//
// SPDX-License-Identifier: EUPL-1.2
// SPDX-FileCopyrightText: 2020-present Detlef Stern
//-----------------------------------------------------------------------------

package markdown

import (
	"strings"
	"testing"

	"zettelstore.de/z/ast"
)

func TestSplitText(t *testing.T) {
	t.Parallel()
	var testcases = []struct {
		text string
		exp  string
	}{
		{"", ""},
		{"abc", "Tabc"},
		{" ", "S "},
		{"abc def", "TabcS Tdef"},
		{"abc def ", "TabcS TdefS "},
		{" abc def ", "S TabcS TdefS "},
	}
	for i, tc := range testcases {
		var sb strings.Builder
		for _, in := range splitText(tc.text) {
			switch n := in.(type) {
			case *ast.TextNode:
				sb.WriteByte('T')
				sb.WriteString(n.Text)
			case *ast.SpaceNode:
				sb.WriteByte('S')
				sb.WriteString(n.Lexeme)
			default:
				sb.WriteByte('Q')
			}
		}
		got := sb.String()
		if tc.exp != got {
			t.Errorf("TC=%d, text=%q, exp=%q, got=%q", i, tc.text, tc.exp, got)
		}
	}
}

func ParseMetadata(value string) ast.InlineSlice {
	return ParseInlines(input.NewInput([]byte(value)), meta.SyntaxZmk)
}

// ParseSpacedText returns an inline slice that consists just of test and space node.
// No Zettelmarkup parsing is done. It is typically used to transform the zettel title into an inline slice.
func ParseSpacedText(s string) ast.InlineSlice {
	return ast.CreateInlineSliceFromWords(meta.ListFromValue(s)...)
}

// NormalizedSpacedText returns the given string, but normalize multiple spaces to one space.
func NormalizedSpacedText(s string) string { return strings.Join(meta.ListFromValue(s), " ") }

// ParseDescription returns a suitable description stored in the metadata as an inline slice.
// This is done for an image in most cases.
func ParseDescription(m *meta.Meta) ast.InlineSlice {
	if m == nil {
		return nil
	}
	if descr, found := m.Get(api.KeySummary); found {
		in := ParseMetadata(descr)
		cleaner.CleanInlineLinks(&in)
		return in
	}
	if title, found := m.Get(api.KeyTitle); found {
		return ParseSpacedText(title)
	}
	return ast.CreateInlineSliceFromWords("Zettel", "without", "title:", m.Zid.String())
}

// ParseZettel parses the zettel based on the syntax.
func ParseZettel(ctx context.Context, zettel zettel.Zettel, syntax string, rtConfig config.Config) *ast.ZettelNode {
	m := zettel.Meta
	inhMeta := m
	if rtConfig != nil {

}

func startsWithSpaceSoftBreak(pn *ast.ParaNode) bool {
	ins := pn.Inlines
	if len(ins) < 2 {
		return false
	}
	_, isSpace := ins[0].(*ast.SpaceNode)
	_, isBreak := ins[1].(*ast.BreakNode)






	return isSpace && isBreak





}

func (cp *zmkP) cleanupListsAfterEOL() {
	for _, l := range cp.lists {
		if lits := len(l.Items); lits > 0 {
			l.Items[lits-1] = append(l.Items[lits-1], &nullItemNode{})
		}

		var in ast.InlineNode
		success := false
		switch inp.Ch {
		case input.EOS:
			return nil
		case '\n', '\r':
			return cp.parseSoftBreak()
		case ' ', '\t':
			return cp.parseSpace()
		case '[':
			inp.Next()
			switch inp.Ch {
			case '[':
				in, success = cp.parseLink()
			case '@':
				in, success = cp.parseCite()

		return cp.parseBackslashRest()
	}
	for {
		inp.Next()
		switch inp.Ch {
		// The following case must contain all runes that occur in parseInline!
		// Plus the closing brackets ] and } and ) and the middle |
		case input.EOS, '\n', '\r', ' ', '\t', '[', ']', '{', '}', '(', ')', '|', '%', '_', '*', '>', '~', '^', ',', '"', '#', ':', '\'', '@', '`', runeModGrave, '$', '=', '\\', '-', '&':
			return &ast.TextNode{Text: string(inp.Src[pos:inp.Pos])}
		}
	}
}

func (cp *zmkP) parseBackslash() ast.InlineNode {
	inp := cp.inp

		return &ast.TextNode{Text: "\u00a0"}
	}
	pos := inp.Pos
	inp.Next()
	return &ast.TextNode{Text: string(inp.Src[pos:inp.Pos])}
}

func (cp *zmkP) parseSpace() *ast.SpaceNode {
	inp := cp.inp
	pos := inp.Pos
	for {
		inp.Next()
		switch inp.Ch {
		case ' ', '\t':
		default:
			return &ast.SpaceNode{Lexeme: string(inp.Src[pos:inp.Pos])}
		}
	}
}

func (cp *zmkP) parseSoftBreak() *ast.BreakNode {
	cp.inp.EatEOL()
	return &ast.BreakNode{}
}

func (cp *zmkP) parseLink() (*ast.LinkNode, bool) {
	if ref, is, ok := cp.parseReference('[', ']'); ok {

func (pp *postProcessor) visitRegion(rn *ast.RegionNode) {
	oldVerse := pp.inVerse
	if rn.Kind == ast.RegionVerse {
		pp.inVerse = true
	}
	pp.visitBlockSlice(&rn.Blocks)

	if len(rn.Inlines) > 0 {
		pp.visitInlineSlice(&rn.Inlines)
	}
	pp.inVerse = oldVerse
}

func (pp *postProcessor) visitNestedList(ln *ast.NestedListNode) {
	for i, item := range ln.Items {
		ln.Items[i] = pp.processItemSlice(item)
	}
	if ln.Kind != ast.NestedListQuote {

}

// processInlineSliceHead removes leading spaces and empty text.
func (pp *postProcessor) processInlineSliceHead(is *ast.InlineSlice) {
	ins := *is
	for i, in := range ins {
		switch in := in.(type) {
		case *ast.SpaceNode:
			if pp.inVerse {
				*is = ins[i:]
				return
			}
		case *ast.TextNode:





			if len(in.Text) > 0 {
				*is = ins[i:]
				return
			}
		default:
			*is = ins[i:]
			return

	ins := *is
	fromPos, toPos := 0, 0
	for fromPos < maxPos {
		ins[toPos] = ins[fromPos]
		fromPos++
		switch in := ins[toPos].(type) {
		case *ast.TextNode:




			fromPos = processTextNode(ins, maxPos, in, fromPos)









		case *ast.SpaceNode:








			if pp.inVerse {
				in.Lexeme = strings.Repeat("\u00a0", in.Count())
			}
			fromPos = processSpaceNode(ins, maxPos, in, toPos, fromPos)
		case *ast.BreakNode:
			if pp.inVerse {
				in.Hard = true
			}
		}
		toPos++
	}
	return toPos
}

func processTextNode(ins ast.InlineSlice, maxPos int, in *ast.TextNode, fromPos int) int {
	for fromPos < maxPos {
		if tn, ok := ins[fromPos].(*ast.TextNode); ok {
			in.Text = in.Text + tn.Text
			fromPos++
		} else {
			break
		}
	}
	return fromPos
}

func processSpaceNode(ins ast.InlineSlice, maxPos int, in *ast.SpaceNode, toPos, fromPos int) int {
	if fromPos < maxPos {
		switch nn := ins[fromPos].(type) {
		case *ast.BreakNode:
			if in.Count() > 1 {
				nn.Hard = true
				ins[toPos] = nn
				fromPos++
			}
		case *ast.LiteralNode:
			if nn.Kind == ast.LiteralComment {
				ins[toPos] = ins[fromPos]
				fromPos++
			}
		}
	}
	return fromPos
}

// processInlineSliceTail removes empty text nodes, breaks and spaces at the end.
func (*postProcessor) processInlineSliceTail(is *ast.InlineSlice, toPos int) int {
	ins := *is
	for toPos > 0 {
		switch n := ins[toPos-1].(type) {
		case *ast.TextNode:

			if len(n.Text) > 0 {
				return toPos
			}
		case *ast.BreakNode:
		case *ast.SpaceNode:
		default:
			return toPos
		}
		toPos--
		ins[toPos] = nil // Kill node to enable garbage collection
	}
	return toPos
}

	})
}

func TestText(t *testing.T) {
	t.Parallel()
	checkTcs(t, TestCases{
		{"abcd", "(PARA abcd)"},
		{"ab cd", "(PARA ab SP cd)"},
		{"abcd ", "(PARA abcd)"},
		{" abcd", "(PARA abcd)"},
		{"\\", "(PARA \\)"},
		{"\\\n", ""},
		{"\\\ndef", "(PARA HB def)"},
		{"\\\r", ""},
		{"\\\rdef", "(PARA HB def)"},
		{"\\\r\n", ""},
		{"\\\r\ndef", "(PARA HB def)"},
		{"\\a", "(PARA a)"},
		{"\\aa", "(PARA aa)"},
		{"a\\a", "(PARA aa)"},
		{"\\+", "(PARA +)"},
		{"\\ ", "(PARA \u00a0)"},
		{"http://a, http://b", "(PARA http://a, SP http://b)"},
	})
}

func TestSpace(t *testing.T) {
	t.Parallel()
	checkTcs(t, TestCases{
		{" ", ""},

		{"[", "(PARA [)"},
		{"[[", "(PARA [[)"},
		{"[[|", "(PARA [[|)"},
		{"[[]", "(PARA [[])"},
		{"[[|]", "(PARA [[|])"},
		{"[[]]", "(PARA [[]])"},
		{"[[|]]", "(PARA [[|]])"},
		{"[[ ]]", "(PARA [[ SP ]])"},
		{"[[\n]]", "(PARA [[ SB ]])"},
		{"[[ a]]", "(PARA (LINK a))"},
		{"[[a ]]", "(PARA [[a SP ]])"},
		{"[[a\n]]", "(PARA [[a SB ]])"},
		{"[[a]]", "(PARA (LINK a))"},
		{"[[12345678901234]]", "(PARA (LINK 12345678901234))"},
		{"[[a]", "(PARA [[a])"},
		{"[[|a]]", "(PARA [[|a]])"},
		{"[[b|]]", "(PARA [[b|]])"},
		{"[[b|a]]", "(PARA (LINK a b))"},
		{"[[b| a]]", "(PARA (LINK a b))"},
		{"[[b%c|a]]", "(PARA (LINK a b%c))"},
		{"[[b%%c|a]]", "(PARA [[b {% c|a]]})"},
		{"[[b|a]", "(PARA [[b|a])"},
		{"[[b\nc|a]]", "(PARA (LINK a b SB c))"},
		{"[[b c|a#n]]", "(PARA (LINK a#n b SP c))"},
		{"[[a]]go", "(PARA (LINK a) go)"},
		{"[[b|a]]{go}", "(PARA (LINK a b)[ATTR go])"},
		{"[[[[a]]|b]]", "(PARA [[ (LINK a) |b]])"},
		{"[[a[b]c|d]]", "(PARA (LINK d a[b]c))"},
		{"[[[b]c|d]]", "(PARA [ (LINK d b]c))"},
		{"[[a[]c|d]]", "(PARA (LINK d a[]c))"},
		{"[[a[b]|d]]", "(PARA (LINK d a[b]))"},

func TestCite(t *testing.T) {
	t.Parallel()
	checkTcs(t, TestCases{
		{"[@", "(PARA [@)"},
		{"[@]", "(PARA [@])"},
		{"[@a]", "(PARA (CITE a))"},
		{"[@ a]", "(PARA [@ SP a])"},
		{"[@a ]", "(PARA (CITE a))"},
		{"[@a\n]", "(PARA (CITE a))"},
		{"[@a\nx]", "(PARA (CITE a SB x))"},
		{"[@a\n\n]", "(PARA [@a)(PARA ])"},
		{"[@a,\n]", "(PARA (CITE a))"},
		{"[@a,n]", "(PARA (CITE a n))"},
		{"[@a| n]", "(PARA (CITE a n))"},

		{"{", "(PARA {)"},
		{"{{", "(PARA {{)"},
		{"{{|", "(PARA {{|)"},
		{"{{}", "(PARA {{})"},
		{"{{|}", "(PARA {{|})"},
		{"{{}}", "(PARA {{}})"},
		{"{{|}}", "(PARA {{|}})"},
		{"{{ }}", "(PARA {{ SP }})"},
		{"{{\n}}", "(PARA {{ SB }})"},
		{"{{a }}", "(PARA {{a SP }})"},
		{"{{a\n}}", "(PARA {{a SB }})"},
		{"{{a}}", "(PARA (EMBED a))"},
		{"{{12345678901234}}", "(PARA (EMBED 12345678901234))"},
		{"{{ a}}", "(PARA (EMBED a))"},
		{"{{a}", "(PARA {{a})"},
		{"{{|a}}", "(PARA {{|a}})"},
		{"{{b|}}", "(PARA {{b|}})"},
		{"{{b|a}}", "(PARA (EMBED a b))"},
		{"{{b| a}}", "(PARA (EMBED a b))"},
		{"{{b|a}", "(PARA {{b|a})"},
		{"{{b\nc|a}}", "(PARA (EMBED a b SB c))"},
		{"{{b c|a#n}}", "(PARA (EMBED a#n b SP c))"},
		{"{{a}}{go}", "(PARA (EMBED a)[ATTR go])"},
		{"{{{{a}}|b}}", "(PARA {{ (EMBED a) |b}})"},
		{"{{\\|}}", "(PARA (EMBED %5C%7C))"},
		{"{{\\||a}}", "(PARA (EMBED a |))"},
		{"{{b\\||a}}", "(PARA (EMBED a b|))"},
		{"{{b\\|c|a}}", "(PARA (EMBED a b|c))"},
		{"{{\\}}}", "(PARA (EMBED %5C%7D))"},

func TestMark(t *testing.T) {
	t.Parallel()
	checkTcs(t, TestCases{
		{"[!", "(PARA [!)"},
		{"[!\n", "(PARA [!)"},
		{"[!]", "(PARA (MARK #*))"},
		{"[!][!]", "(PARA (MARK #*) (MARK #*-1))"},
		{"[! ]", "(PARA [! SP ])"},
		{"[!a]", "(PARA (MARK \"a\" #a))"},
		{"[!a][!a]", "(PARA (MARK \"a\" #a) (MARK \"a\" #a-1))"},
		{"[!a ]", "(PARA [!a SP ])"},
		{"[!a_]", "(PARA (MARK \"a_\" #a))"},
		{"[!a_][!a]", "(PARA (MARK \"a_\" #a) (MARK \"a\" #a-1))"},
		{"[!a-b]", "(PARA (MARK \"a-b\" #a-b))"},
		{"[!a|b]", "(PARA (MARK \"a\" #a b))"},
		{"[!a|]", "(PARA (MARK \"a\" #a))"},
		{"[!|b]", "(PARA (MARK #* b))"},
		{"[!|b c]", "(PARA (MARK #* b SP c))"},
	})
}

func TestComment(t *testing.T) {
	t.Parallel()
	checkTcs(t, TestCases{
		{"%", "(PARA %)"},

		// Good cases
		{"&lt;", "(PARA <)"},
		{"&#48;", "(PARA 0)"},
		{"&#x4A;", "(PARA J)"},
		{"&#X4a;", "(PARA J)"},
		{"&hellip;", "(PARA \u2026)"},
		{"&nbsp;", "(PARA \u00a0)"},
		{"E: &amp;,&#63;;&#x63;.", "(PARA E: SP &,?;c.)"},
	})
}

func TestVerbatimZettel(t *testing.T) {
	t.Parallel()
	checkTcs(t, TestCases{
		{"@@@\n@@@", "(ZETTEL)"},

	}))
}

func TestHeading(t *testing.T) {
	t.Parallel()
	checkTcs(t, TestCases{
		{"=h", "(PARA =h)"},
		{"= h", "(PARA = SP h)"},
		{"==h", "(PARA ==h)"},
		{"== h", "(PARA == SP h)"},
		{"===h", "(PARA ===h)"},
		{"=== h", "(H1 h #h)"},
		{"===  h", "(H1 h #h)"},
		{"==== h", "(H2 h #h)"},
		{"===== h", "(H3 h #h)"},
		{"====== h", "(H4 h #h)"},
		{"======= h", "(H5 h #h)"},
		{"======== h", "(H5 h #h)"},
		{"=", "(PARA =)"},
		{"=== h=__=a__", "(H1 h= {_ =a} #h-a)"},
		{"=\n", "(PARA =)"},
		{"a=", "(PARA a=)"},
		{" =", "(PARA =)"},
		{"=== h\na", "(H1 h #h)(PARA a)"},
		{"=== h i {-}", "(H1 h SP i #h-i)[ATTR -]"},
		{"=== h {{a}}", "(H1 h SP (EMBED a) #h)"},
		{"=== h{{a}}", "(H1 h (EMBED a) #h)"},
		{"=== {{a}}", "(H1 (EMBED a))"},
		{"=== h {{a}}{-}", "(H1 h SP (EMBED a)[ATTR -] #h)"},
		{"=== h {{a}} {-}", "(H1 h SP (EMBED a) #h)[ATTR -]"},
		{"=== h {-}{{a}}", "(H1 h #h)[ATTR -]"},
		{"=== h{id=abc}", "(H1 h #h)[ATTR id=abc]"},
		{"=== h\n=== h", "(H1 h #h)(H1 h #h-1)"},
	})
}

func TestHRule(t *testing.T) {

		{">", "(QL {})"},
	})
}

func TestQuoteList(t *testing.T) {
	t.Parallel()
	checkTcs(t, TestCases{
		{"> w1 w2", "(QL {(PARA w1 SP w2)})"},
		{"> w1\n> w2", "(QL {(PARA w1 SB w2)})"},
		{"> w1\n>\n>w2", "(QL {(PARA w1)} {})(PARA >w2)"},
	})
}

func TestEnumAfterPara(t *testing.T) {
	t.Parallel()

		{"; ", "(PARA ;)"},
		{"; abc", "(DL (DT abc))"},
		{"; abc\ndef", "(DL (DT abc))(PARA def)"},
		{"; abc\n def", "(DL (DT abc))(PARA def)"},
		{"; abc\n  def", "(DL (DT abc SB def))"},
		{":", "(PARA :)"},
		{": ", "(PARA :)"},
		{": abc", "(PARA : SP abc)"},
		{"; abc\n: def", "(DL (DT abc) (DD (PARA def)))"},
		{"; abc\n: def\nghi", "(DL (DT abc) (DD (PARA def)))(PARA ghi)"},
		{"; abc\n: def\n ghi", "(DL (DT abc) (DD (PARA def)))(PARA ghi)"},
		{"; abc\n: def\n  ghi", "(DL (DT abc) (DD (PARA def SB ghi)))"},
		{"; abc\n: def\n\n  ghi", "(DL (DT abc) (DD (PARA def)(PARA ghi)))"},
		{"; abc\n:", "(DL (DT abc))(PARA :)"},
		{"; abc\n: def\n: ghi", "(DL (DT abc) (DD (PARA def)) (DD (PARA ghi)))"},

	})
	checkTcs(t, replace("\"", TestCases{
		{"::a::{py=3}", "(PARA {: a}[ATTR py=3])"},
		{"::a::{py=$2 3$}", "(PARA {: a}[ATTR py=$2 3$])"},
		{"::a::{py=$2\\$3$}", "(PARA {: a}[ATTR py=2$3])"},
		{"::a::{py=2$3}", "(PARA {: a}[ATTR py=2$3])"},
		{"::a::{py=$2\n3$}", "(PARA {: a}[ATTR py=$2\n3$])"},
		{"::a::{py=$2 3}", "(PARA {: a} {py=$2 SP 3})"},

		{"::a::{py=2 py=3}", "(PARA {: a}[ATTR py=$2 3$])"},
		{"::a::{.go .py}", "(PARA {: a}[ATTR class=$go py$])"},
	}))
}

func TestTemp(t *testing.T) {

		tv.visitAttributes(n.Attrs)
	case *ast.BLOBNode:
		tv.sb.WriteString("(BLOB ")
		tv.sb.WriteString(n.Syntax)
		tv.sb.WriteString(")")
	case *ast.TextNode:
		tv.sb.WriteString(n.Text)
	case *ast.SpaceNode:
		if l := n.Count(); l == 1 {
			tv.sb.WriteString("SP")
		} else {
			fmt.Fprintf(&tv.sb, "SP%d", l)
		}
	case *ast.BreakNode:
		if n.Hard {
			tv.sb.WriteString("HB")
		} else {
			tv.sb.WriteString("SB")
		}
	case *ast.LinkNode:

	old[n-1].meta = nil // avoid memory leak
	*q = old[0 : n-1]
	return item
}

type contextTask struct {
	port     ContextPort
	seen     id.Set
	queue    ztlCtxQueue
	maxCost  float64
	limit    int
	tagMetas map[string][]*meta.Meta
	tagZids  map[string]id.Set     // just the zids of tagMetas
	metaZid  map[id.Zid]*meta.Meta // maps zid to meta for all meta retrieved with tags
}

func newQueue(startSeq []*meta.Meta, maxCost float64, limit int, port ContextPort) *contextTask {
	result := &contextTask{
		port:     port,
		seen:     id.NewSet(),
		maxCost:  maxCost,
		limit:    limit,
		tagMetas: make(map[string][]*meta.Meta),
		tagZids:  make(map[string]id.Set),
		metaZid:  make(map[id.Zid]*meta.Meta),
	}

	queue := make(ztlCtxQueue, 0, len(startSeq))
	for _, m := range startSeq {
		queue = append(queue, ztlCtxItem{cost: 1, meta: m})
	}

	}
}

func (ct *contextTask) addMeta(m *meta.Meta, newCost float64) {
	// If len(zc.seen) <= 1, the initial zettel is processed. In this case allow all
	// other zettel that are directly reachable, without taking the cost into account.
	// Of course, the limit ist still relevant.
	if !ct.hasLimit() && (len(ct.seen) <= 1 || ct.maxCost == 0 || newCost <= ct.maxCost) {
		if _, found := ct.seen[m.Zid]; !found {
			heap.Push(&ct.queue, ztlCtxItem{cost: newCost, meta: m})
		}
	}
}

func (ct *contextTask) addIDSet(ctx context.Context, newCost float64, value string) {
	elems := meta.ListFromValue(value)
	refCost := referenceCost(newCost, len(elems))
	for _, val := range elems {
		ct.addID(ctx, refCost, val)
	}
}

func referenceCost(baseCost float64, numReferences int) float64 {
	nRefs := float64(numReferences)
	return nRefs*math.Log2(nRefs+1) + baseCost
}

func (ct *contextTask) addTags(ctx context.Context, tags []string, baseCost float64) {
	var zidSet id.Set
	for _, tag := range tags {
		zs := ct.updateTagData(ctx, tag)
		zidSet = zidSet.Copy(zs)
	}
	for _, zid := range zidSet.Sorted() { // .Sorted() to stay deterministic
		minCost := math.MaxFloat64
		costFactor := 1.1
		for _, tag := range tags {
			tagZids := ct.tagZids[tag]
			if tagZids.Contains(zid) {
				cost := tagCost(baseCost, len(tagZids))
				if cost < minCost {
					minCost = cost
				}
				costFactor /= 1.1
			}
		}
		ct.addMeta(ct.metaZid[zid], minCost*costFactor)
	}
}

func (ct *contextTask) updateTagData(ctx context.Context, tag string) id.Set {
	if _, found := ct.tagMetas[tag]; found {
		return ct.tagZids[tag]
	}
	q := Parse(api.KeyTags + api.SearchOperatorHas + tag + " ORDER REVERSE " + api.KeyID)
	ml, err := ct.port.SelectMeta(ctx, nil, q)
	if err != nil {
		ml = nil

	if ct.hasLimit() {
		return nil, -1
	}
	for len(ct.queue) > 0 {
		item := heap.Pop(&ct.queue).(ztlCtxItem)
		m := item.meta
		zid := m.Zid
		if _, found := ct.seen[zid]; found {
			continue
		}
		ct.seen.Add(zid)
		return m, item.cost
	}
	return nil, -1
}

func (ct *contextTask) hasLimit() bool {
	limit := ct.limit
	return limit > 0 && len(ct.seen) >= limit
}

	for {
		pos := inp.Pos
		zid, found := ps.scanZid()
		if !found {
			inp.SetPos(pos)
			break
		}
		if !zidSet.ContainsOrNil(zid) {
			zidSet.Add(zid)
			q = createIfNeeded(q)
			q.zids = append(q.zids, zid)
		}
		ps.skipSpace()
		if ps.mustStop() {
			q.zids = nil

	"zettelstore.de/z/zettel/meta"
)

// Searcher is used to select zettel identifier based on search criteria.
type Searcher interface {
	// Select all zettel that contains the given exact word.
	// The word must be normalized through Unicode NKFD, trimmed and not empty.
	SearchEqual(word string) id.Set

	// Select all zettel that have a word with the given prefix.
	// The prefix must be normalized through Unicode NKFD, trimmed and not empty.
	SearchPrefix(prefix string) id.Set

	// Select all zettel that have a word with the given suffix.
	// The suffix must be normalized through Unicode NKFD, trimmed and not empty.
	SearchSuffix(suffix string) id.Set

	// Select all zettel that contains the given string.
	// The string must be normalized through Unicode NKFD, trimmed and not empty.
	SearchContains(s string) id.Set
}

// Query specifies a mechanism for querying zettel.
type Query struct {
	// Präfixed zettel identifier.
	zids []id.Zid

			cTerm.Match = matchAlways
		}
		result.Terms = append(result.Terms, cTerm)
	}
	return result
}

func metaList2idSet(ml []*meta.Meta) id.Set {
	if ml == nil {
		return nil
	}
	result := id.NewSetCap(len(ml))
	for _, m := range ml {
		result = result.Add(m.Zid)
	}
	return result
}

func (ct *conjTerms) retrieveAndCompileTerm(searcher Searcher, startSet id.Set) CompiledTerm {
	match := ct.compileMeta() // Match might add some searches
	var pred RetrievePredicate
	if searcher != nil {
		pred = ct.retrieveIndex(searcher)
		if startSet != nil {
			if pred == nil {
				pred = startSet.ContainsOrNil
			} else {
				predSet := id.NewSetCap(len(startSet))
				for zid := range startSet {
					if pred(zid) {
						predSet = predSet.Add(zid)
					}
				}
				pred = predSet.ContainsOrNil
			}
		}
	}
	return CompiledTerm{Match: match, Retrieve: pred}
}

	positives := retrievePositives(normCalls, plainCalls)
	if positives == nil {
		// No positive search for words, must contain only words for a negative search.
		// Otherwise len(search) == 0 (see above)
		negatives := retrieveNegatives(negCalls)
		return func(zid id.Zid) bool { return !negatives.ContainsOrNil(zid) }
	}
	if len(positives) == 0 {
		// Positive search didn't found anything. We can omit the negative search.
		return neverIncluded
	}
	if len(negCalls) == 0 {
		// Positive search found something, but there is no negative search.
		return positives.ContainsOrNil
	}
	negatives := retrieveNegatives(negCalls)
	if negatives == nil {
		return positives.ContainsOrNil
	}
	return func(zid id.Zid) bool {
		return positives.ContainsOrNil(zid) && !negatives.ContainsOrNil(zid)
	}
}

// Limit returns only s.GetLimit() elements of the given list.
func (q *Query) Limit(metaList []*meta.Meta) []*meta.Meta {
	if q == nil {
		return metaList
	}
	return limitElements(metaList, q.limit)
}

	"zettelstore.de/z/zettel/id"
)

type searchOp struct {
	s  string
	op compareOp
}
type searchFunc func(string) id.Set
type searchCallMap map[searchOp]searchFunc

var cmpPred = map[compareOp]func(string, string) bool{
	cmpEqual:   stringEqual,
	cmpPrefix:  strings.HasPrefix,
	cmpSuffix:  strings.HasSuffix,
	cmpMatch:   strings.Contains,

		if _, found := plainCalls[val]; found {
			return true
		}
	}
	return false
}

func retrievePositives(normCalls, plainCalls searchCallMap) id.Set {
	if isSuperset(normCalls, plainCalls) {
		var normResult id.Set
		for c, sf := range normCalls {
			normResult = normResult.IntersectOrSet(sf(c.s))
		}
		return normResult
	}

	type searchResults map[searchOp]id.Set
	var cache searchResults
	var plainResult id.Set
	for c, sf := range plainCalls {
		result := sf(c.s)
		if _, found := normCalls[c]; found {
			if cache == nil {
				cache = make(searchResults)
			}
			cache[c] = result
		}
		plainResult = plainResult.IntersectOrSet(result)
	}
	var normResult id.Set
	for c, sf := range normCalls {
		if cache != nil {
			if result, found := cache[c]; found {
				normResult = normResult.IntersectOrSet(result)
				continue
			}
		}
		normResult = normResult.IntersectOrSet(sf(c.s))
	}
	return normResult.Copy(plainResult)
}

func isSuperset(normCalls, plainCalls searchCallMap) bool {
	for c := range plainCalls {
		if _, found := normCalls[c]; !found {
			return false
		}
	}
	return true
}

func retrieveNegatives(negCalls searchCallMap) id.Set {
	var negatives id.Set
	for val, sf := range negCalls {
		negatives = negatives.Copy(sf(val.s))
	}
	return negatives
}

func getSearchFunc(searcher Searcher, op compareOp) searchFunc {
	switch op {
	case cmpEqual:

		if got := strfun.Slugify(test.in); got != test.exp {
			t.Errorf("%q: %q != %q", test.in, got, test.exp)
		}
	}
}

func eqStringSlide(got, exp []string) bool {



	if len(got) != len(exp) {
		return false
	}
	for i, g := range got {
		if g != exp[i] {
			return false
		}
	}
	return true
}

func TestNormalizeWord(t *testing.T) {
	t.Parallel()
	tests := []struct {
		in  string
		exp []string
	}{
		{"", []string{}},
		{" ", []string{}},
		{"ˋ", []string{}}, // No single diacritic char, such as U+02CB
		{"simple test", []string{"simple", "test"}},
		{"I'm a go developer", []string{"i", "m", "a", "go", "developer"}},
		{"-!->simple   test<-!-", []string{"simple", "test"}},
		{"äöüÄÖÜß", []string{"aouaouß"}},
		{"\"aèf", []string{"aef"}},
		{"a#b", []string{"a", "b"}},
		{"*", []string{}},
		{"123", []string{"123"}},
		{"1²3", []string{"123"}},
		{"Period.", []string{"period"}},
		{" WORD  NUMBER ", []string{"word", "number"}},
	}
	for _, test := range tests {
		if got := strfun.NormalizeWords(test.in); !eqStringSlide(got, test.exp) {
			t.Errorf("%q: %q != %q", test.in, got, test.exp)
		}
	}
}

			}
		}
	}
	candidates = filterByZid(candidates, refZids)
	return uc.filterCandidates(ctx, candidates, words)
}

func filterByZid(candidates []*meta.Meta, ignoreSeq id.Set) []*meta.Meta {
	result := make([]*meta.Meta, 0, len(candidates))
	for _, m := range candidates {
		if !ignoreSeq.ContainsOrNil(m.Zid) {
			result = append(result, m)
		}
	}
	return result
}

func (uc *Query) filterCandidates(ctx context.Context, candidates []*meta.Meta, words []string) []*meta.Meta {

func (v *unlinkedVisitor) splitInlineTextList(is *ast.InlineSlice) []string {
	var result []string
	var curList []string
	for _, in := range *is {
		switch n := in.(type) {
		case *ast.TextNode:
			curList = append(curList, strfun.MakeWords(n.Text)...)
		case *ast.SpaceNode:
		default:
			if curList != nil {
				result = append(result, v.joinWords(curList))
				curList = nil
			}
		}
	}
	if curList != nil {
		result = append(result, v.joinWords(curList))
	}
	return result
}

type getMetaFunc func(context.Context, id.Zid) (*meta.Meta, error)

func buildSxnCodeDigraph(ctx context.Context, startZid id.Zid, getMeta getMetaFunc) id.Digraph {
	m, err := getMeta(ctx, startZid)
	if err != nil {
		return nil
	}
	var marked id.Set
	stack := []*meta.Meta{m}
	dg := id.Digraph(nil).AddVertex(startZid)
	for pos := len(stack) - 1; pos >= 0; pos = len(stack) - 1 {
		curr := stack[pos]
		stack = stack[:pos]
		if marked.Contains(curr.Zid) {
			continue

import (
	"maps"
	"slices"
)

// Digraph relates zettel identifier in a directional way.
type Digraph map[Zid]Set

// AddVertex adds an edge / vertex to the digraph.
func (dg Digraph) AddVertex(zid Zid) Digraph {
	if dg == nil {
		return Digraph{zid: nil}
	}
	if _, found := dg[zid]; !found {

	}
}

// AddEdge adds a connection from `zid1` to `zid2`.
// Both vertices must be added before. Otherwise the function may panic.
func (dg Digraph) AddEdge(fromZid, toZid Zid) Digraph {
	if dg == nil {
		return Digraph{fromZid: Set(nil).Add(toZid), toZid: nil}
	}
	dg[fromZid] = dg[fromZid].Add(toZid)
	return dg
}

// AddEgdes adds all given `Edge`s to the digraph.
//

		dg = dg.AddEdge(edge.From, edge.To)
	}
	return dg
}

// Equal returns true if both digraphs have the same vertices and edges.
func (dg Digraph) Equal(other Digraph) bool {
	return maps.EqualFunc(dg, other, func(cg, co Set) bool { return cg.Equal(co) })
}

// Clone a digraph.
func (dg Digraph) Clone() Digraph {
	if len(dg) == 0 {
		return nil
	}

		return false
	}
	_, found := dg[zid]
	return found
}

// Vertices returns the set of all vertices.
func (dg Digraph) Vertices() Set {
	if len(dg) == 0 {
		return nil
	}
	verts := NewSetCap(len(dg))
	for vert := range dg {
		verts.Add(vert)
	}
	return verts
}

// Edges returns an unsorted slice of the edges of the digraph.
func (dg Digraph) Edges() (es EdgeSlice) {
	for vert, closure := range dg {
		for next := range closure {
			es = append(es, Edge{From: vert, To: next})
		}
	}
	return es
}

// Originators will return the set of all vertices that are not referenced
// a the to-part of an edge.
func (dg Digraph) Originators() Set {
	if len(dg) == 0 {
		return nil
	}
	origs := dg.Vertices()
	for _, closure := range dg {
		origs.Substract(closure)
	}
	return origs
}

// Terminators returns the set of all vertices that does not reference
// other vertices.
func (dg Digraph) Terminators() (terms Set) {
	for vert, closure := range dg {
		if len(closure) == 0 {
			terms = terms.Add(vert)
		}
	}
	return terms
}

// TransitiveClosure calculates the sub-graph that is reachable from `zid`.
func (dg Digraph) TransitiveClosure(zid Zid) (tc Digraph) {
	if len(dg) == 0 {
		return nil
	}
	var marked Set
	stack := Slice{zid}
	for pos := len(stack) - 1; pos >= 0; pos = len(stack) - 1 {
		curr := stack[pos]
		stack = stack[:pos]
		if marked.Contains(curr) {
			continue
		}
		tc = tc.AddVertex(curr)
		for next := range dg[curr] {
			tc = tc.AddVertex(next)
			tc = tc.AddEdge(curr, next)
			stack = append(stack, next)
		}
		marked = marked.Add(curr)
	}
	return tc
}

// ReachableVertices calculates the set of all vertices that are reachable
// from the given `zid`.
func (dg Digraph) ReachableVertices(zid Zid) (tc Set) {
	if len(dg) == 0 {
		return nil
	}
	stack := dg[zid].Sorted()
	for last := len(stack) - 1; last >= 0; last = len(stack) - 1 {
		curr := stack[last]
		stack = stack[:last]
		if tc.Contains(curr) {
			continue
		}
		closure, found := dg[curr]
		if !found {
			continue
		}
		tc = tc.Add(curr)
		for next := range closure {
			stack = append(stack, next)
		}
	}
	return tc
}

// IsDAG returns a vertex and false, if the graph has a cycle containing the vertex.
func (dg Digraph) IsDAG() (Zid, bool) {
	for vertex := range dg {
		if dg.ReachableVertices(vertex).Contains(vertex) {
			return vertex, false
		}
	}
	return Invalid, true
}

// Reverse returns a graph with reversed edges.
func (dg Digraph) Reverse() (revDg Digraph) {
	for vertex, closure := range dg {
		revDg = revDg.AddVertex(vertex)
		for next := range closure {
			revDg = revDg.AddVertex(next)
			revDg = revDg.AddEdge(next, vertex)
		}
	}
	return revDg
}

// SortReverse returns a deterministic, topological, reverse sort of the
// digraph.
//
// Works only if digraph is a DAG. Otherwise the algorithm will not terminate
// or returns an arbitrary value.
func (dg Digraph) SortReverse() (sl Slice) {
	if len(dg) == 0 {
		return nil
	}
	tempDg := dg.Clone()
	for len(tempDg) > 0 {
		terms := tempDg.Terminators()
		if len(terms) == 0 {
			break
		}
		termSlice := terms.Sorted()
		slices.Reverse(termSlice)
		sl = append(sl, termSlice...)
		for t := range terms {
			tempDg.RemoveVertex(t)
		}
	}
	return sl
}

}

func TestDigraphOriginators(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		name string
		dg   id.EdgeSlice
		orig id.Set
		term id.Set
	}{
		{"empty", nil, nil, nil},
		{"single", zps{{0, 1}}, id.NewSet(0), id.NewSet(1)},
		{"chain", zps{{0, 1}, {1, 2}, {2, 3}}, id.NewSet(0), id.NewSet(3)},
	}
	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {

func TestDigraphReachableVertices(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		name  string
		pairs id.EdgeSlice
		start id.Zid
		exp   id.Set
	}{
		{"nil", nil, 0, nil},
		{"0-2", zps{{1, 2}, {2, 3}}, 1, id.NewSet(2, 3)},
		{"1,2", zps{{1, 2}, {2, 3}}, 2, id.NewSet(3)},
		{"0-2,1-2", zps{{1, 2}, {2, 3}, {1, 3}}, 1, id.NewSet(2, 3)},
		{"0-2,1-2/1", zps{{1, 2}, {2, 3}, {1, 3}}, 2, id.NewSet(3)},
		{"0-2,1-2/2", zps{{1, 2}, {2, 3}, {1, 3}}, 3, nil},

// SPDX-License-Identifier: EUPL-1.2
// SPDX-FileCopyrightText: 2021-present Detlef Stern
//-----------------------------------------------------------------------------

package id

import (
	"maps"
	"strings"
)

// Set is a set of zettel identifier
type Set map[Zid]struct{}



// String returns a string representation of the map.
func (s Set) String() string {





	if s == nil {
		return "{}"
	}
	var sb strings.Builder
	sb.WriteByte('{')
	for i, zid := range s.Sorted() {
		if i > 0 {
			sb.WriteByte(' ')
		}
		sb.Write(zid.Bytes())
	}
	sb.WriteByte('}')
	return sb.String()
}

// NewSet returns a new set of identifier with the given initial values.
func NewSet(zids ...Zid) Set {
	l := len(zids)
	if l < 8 {

		l = 8
	}


	result := make(Set, l)
	result.CopySlice(zids)
	return result
}


// NewSetCap returns a new set of identifier with the given capacity and initial values.
func NewSetCap(c int, zids ...Zid) Set {

	l := len(zids)
	if c < l {
		c = l
	}
	if c < 8 {

		c = 8

	}
	result := make(Set, c)



	result.CopySlice(zids)

	return result
}

// Clone returns a copy of the given set.
func (s Set) Clone() Set {
	if len(s) == 0 {
		return nil
	}
	return maps.Clone(s)
}

// Add adds a Add to the set.
func (s Set) Add(zid Zid) Set {
	if s == nil {
		return NewSet(zid)
	}
	s[zid] = struct{}{}
	return s
}

// Contains return true if the set is non-nil and the set contains the given Zettel identifier.
func (s Set) Contains(zid Zid) bool {
	if s != nil {
		_, found := s[zid]
		return found
	}
	return false
}

// ContainsOrNil return true if the set is nil or if the set contains the given Zettel identifier.
func (s Set) ContainsOrNil(zid Zid) bool {
	if s != nil {
		_, found := s[zid]
		return found
	}
	return true
}

// Copy adds all member from the other set.
func (s Set) Copy(other Set) Set {
	if s == nil {
		if len(other) == 0 {
			return nil
		}
		s = NewSetCap(len(other))
	}
	maps.Copy(s, other)
	return s
}

// CopySlice adds all identifier of the given slice to the set.
func (s Set) CopySlice(sl Slice) Set {
	if s == nil {
		s = NewSetCap(len(sl))
	}

	for _, zid := range sl {
		s[zid] = struct{}{}
	}
	return s
}

// Sorted returns the set as a sorted slice of zettel identifier.
func (s Set) Sorted() Slice {
	if l := len(s); l > 0 {
		result := make(Slice, 0, l)
		for zid := range s {
			result = append(result, zid)
		}
		result.Sort()
		return result
	}
	return nil
}

// IntersectOrSet removes all zettel identifier that are not in the other set.
// Both sets can be modified by this method. One of them is the set returned.
// It contains the intersection of both, if s is not nil.
//
// If s == nil, then the other set is always returned.
func (s Set) IntersectOrSet(other Set) Set {
	if s == nil {
		return other
	}

	if len(s) > len(other) {
		s, other = other, s



	}
	for zid := range s {



		_, otherOk := other[zid]

		if !otherOk {



			delete(s, zid)
		}
	}



	return s
}




// Substract removes all zettel identifier from 's' that are in the set 'other'.
func (s Set) Substract(other Set) {
	if s == nil || other == nil {
		return
	}





















	for zid := range other {
















		delete(s, zid)






	}















}

// Remove the identifier from the set.
func (s Set) Remove(zid Zid) Set {
	if len(s) == 0 {
		return nil
	}
	delete(s, zid)



	if len(s) == 0 {
		return nil
	}
	return s
}

// Equal returns true if the other set is equal to the given set.
func (s Set) Equal(other Set) bool { return maps.Equal(s, other) }

import (
	"testing"

	"zettelstore.de/z/zettel/id"
)

func TestSetContains(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		s   id.Set
		zid id.Zid
		exp bool
	}{
		{nil, id.Invalid, true},
		{nil, 14, true},
		{id.NewSet(), id.Invalid, false},
		{id.NewSet(), 1, false},
		{id.NewSet(), id.Invalid, false},
		{id.NewSet(1), 1, true},
	}
	for i, tc := range testcases {
		got := tc.s.ContainsOrNil(tc.zid)
		if got != tc.exp {
			t.Errorf("%d: %v.Contains(%v) == %v, but got %v", i, tc.s, tc.zid, tc.exp, got)
		}
	}
}

func TestSetAdd(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		s1, s2 id.Set
		exp    id.Slice
	}{
		{nil, nil, nil},
		{id.NewSet(), nil, nil},
		{id.NewSet(), id.NewSet(), nil},
		{nil, id.NewSet(1), id.Slice{1}},
		{id.NewSet(1), nil, id.Slice{1}},
		{id.NewSet(1), id.NewSet(), id.Slice{1}},
		{id.NewSet(1), id.NewSet(2), id.Slice{1, 2}},
		{id.NewSet(1), id.NewSet(1), id.Slice{1}},
	}
	for i, tc := range testcases {
		sl1 := tc.s1.Sorted()
		sl2 := tc.s2.Sorted()
		got := tc.s1.Copy(tc.s2).Sorted()
		if !got.Equal(tc.exp) {
			t.Errorf("%d: %v.Add(%v) should be %v, but got %v", i, sl1, sl2, tc.exp, got)
		}
	}
}

func TestSetSorted(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		set id.Set
		exp id.Slice
	}{
		{nil, nil},
		{id.NewSet(), nil},
		{id.NewSet(9, 4, 6, 1, 7), id.Slice{1, 4, 6, 7, 9}},
	}
	for i, tc := range testcases {
		got := tc.set.Sorted()
		if !got.Equal(tc.exp) {
			t.Errorf("%d: %v.Sorted() should be %v, but got %v", i, tc.set, tc.exp, got)
		}
	}
}

func TestSetIntersectOrSet(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		s1, s2 id.Set
		exp    id.Slice
	}{
		{nil, nil, nil},
		{id.NewSet(), nil, nil},
		{nil, id.NewSet(), nil},
		{id.NewSet(), id.NewSet(), nil},
		{id.NewSet(1), nil, nil},
		{nil, id.NewSet(1), id.Slice{1}},
		{id.NewSet(1), id.NewSet(), nil},
		{id.NewSet(), id.NewSet(1), nil},
		{id.NewSet(1), id.NewSet(2), nil},
		{id.NewSet(2), id.NewSet(1), nil},
		{id.NewSet(1), id.NewSet(1), id.Slice{1}},
	}
	for i, tc := range testcases {
		sl1 := tc.s1.Sorted()
		sl2 := tc.s2.Sorted()
		got := tc.s1.IntersectOrSet(tc.s2).Sorted()
		if !got.Equal(tc.exp) {
			t.Errorf("%d: %v.IntersectOrSet(%v) should be %v, but got %v", i, sl1, sl2, tc.exp, got)
		}
	}
}






























































































func TestSetRemove(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		s1, s2 id.Set
		exp    id.Slice
	}{
		{nil, nil, nil},
		{id.NewSet(), nil, nil},
		{id.NewSet(), id.NewSet(), nil},
		{id.NewSet(1), nil, id.Slice{1}},
		{id.NewSet(1), id.NewSet(), id.Slice{1}},
		{id.NewSet(1), id.NewSet(2), id.Slice{1}},
		{id.NewSet(1), id.NewSet(1), id.Slice{}},
	}
	for i, tc := range testcases {
		sl1 := tc.s1.Sorted()
		sl2 := tc.s2.Sorted()
		newS1 := id.NewSet(sl1...)
		newS1.Substract(tc.s2)
		got := newS1.Sorted()
		if !got.Equal(tc.exp) {
			t.Errorf("%d: %v.Remove(%v) should be %v, but got %v", i, sl1, sl2, tc.exp, got)
		}
	}
}

//	func BenchmarkSet(b *testing.B) {
//		s := id.Set{}
//		for range b.N {
//			s[id.Zid(i)] = true
//		}
//	}
func BenchmarkSet(b *testing.B) {
	s := id.Set{}
	for i := range b.N {
		s[id.Zid(i)] = struct{}{}
	}
}

// Clone a zettel identifier slice
func (zs Slice) Clone() Slice { return slices.Clone(zs) }

// Equal reports whether zs and other are the same length and contain the samle zettel
// identifier. A nil argument is equivalent to an empty slice.
func (zs Slice) Equal(other Slice) bool { return slices.Equal(zs, other) }


func (zs Slice) String() string {
	if len(zs) == 0 {
		return ""
	}
	var sb strings.Builder
	for i, zid := range zs {
		if i > 0 {
			sb.WriteByte(' ')

		got = tc.s2.Equal(tc.s1)
		if got != tc.exp {
			t.Errorf("%d/%v.Equal(%v)==%v, but got %v", i, tc.s2, tc.s1, tc.exp, got)
		}
	}
}

func TestSliceString(t *testing.T) {
	t.Parallel()
	testcases := []struct {
		in  id.Slice
		exp string
	}{
		{nil, ""},
		{id.Slice{}, ""},
		{id.Slice{1}, "00000000000001"},
		{id.Slice{1, 2}, "00000000000001 00000000000002"},
	}
	for i, tc := range testcases {
		got := tc.in.String()
		if got != tc.exp {
			t.Errorf("%d/%v: expected %q, but got %q", i, tc.in, tc.exp, got)
		}
	}
}