ADDED .github/dependabot.yml Index: .github/dependabot.yml ================================================================== --- /dev/null +++ .github/dependabot.yml @@ -0,0 +1,12 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "gomod" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" + rebase-strategy: "disabled" Index: README.md ================================================================== --- README.md +++ README.md @@ -1,3 +1,20 @@ -# zettelstore +# Zettelstore + +**Zettelstore** is a software that collects and relates your notes +(“zettel”) to represent and enhance your knowledge. It helps with +many tasks of personal knowledge management by explicitly supporting the +[Zettelkasten method](https://en.wikipedia.org/wiki/Zettelkasten). The method +is based on creating many individual notes, each with one idea or information, +that are related to each other. Since knowledge is typically build up +gradually, one major focus is a long-term store of these notes, hence the name +“Zettelstore”. + +To get an initial impression, take a look at the +[manual](https://zettelstore.de/manual/). It is a live example of the +zettelstore software, running in read-only mode. + +The software, including the manual, is licensed +under the [European Union Public License 1.2 (or +later)](https://zettelstore.de/home/file?name=LICENSE.txt&ci=trunk). -A storage and service for zettel notes. +[Stay tuned](https://twitter.com/zettelstore)… Index: VERSION ================================================================== --- VERSION +++ VERSION @@ -1,1 +1,1 @@ -0.0.10 +0.0.11 Index: ast/ast.go ================================================================== --- ast/ast.go +++ ast/ast.go @@ -20,15 +20,16 @@ ) // ZettelNode is the root node of the abstract syntax tree. // It is *not* part of the visitor pattern. type ZettelNode struct { - Zettel domain.Zettel - Zid id.Zid // Zettel identification. - InhMeta *meta.Meta // Meta data of the zettel, with inherited values. - Title InlineSlice // Zettel title is a sequence of inline nodes. - Ast BlockSlice // Zettel abstract syntax tree is a sequence of block nodes. + // Zettel domain.Zettel + Meta *meta.Meta // Original metadata + Content domain.Content // Original content + Zid id.Zid // Zettel identification. + InhMeta *meta.Meta // Metadata of the zettel, with inherited values. + Ast BlockSlice // Zettel abstract syntax tree is a sequence of block nodes. } // Node is the interface, all nodes must implement. type Node interface { Accept(v Visitor) Index: ast/block.go ================================================================== --- ast/block.go +++ ast/block.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -188,11 +188,11 @@ // Accept a visitor and visit the node. func (tn *TableNode) Accept(v Visitor) { v.VisitTable(tn) } //-------------------------------------------------------------------------- -// BLOBNode contains just binary data that must be interpreted accordung to +// BLOBNode contains just binary data that must be interpreted according to // a syntax. type BLOBNode struct { Title string Syntax string Blob []byte Index: ast/inline.go ================================================================== --- ast/inline.go +++ ast/inline.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations Index: ast/traverser.go ================================================================== --- ast/traverser.go +++ ast/traverser.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -29,17 +29,17 @@ // VisitRegion traverses the content and the additional text. func (t TopDownTraverser) VisitRegion(rn *RegionNode) { t.v.VisitRegion(rn) t.VisitBlockSlice(rn.Blocks) - t.visitInlineSlice(rn.Inlines) + t.VisitInlineSlice(rn.Inlines) } // VisitHeading traverses the heading. func (t TopDownTraverser) VisitHeading(hn *HeadingNode) { t.v.VisitHeading(hn) - t.visitInlineSlice(hn.Inlines) + t.VisitInlineSlice(hn.Inlines) } // VisitHRule traverses nothing. func (t TopDownTraverser) VisitHRule(hn *HRuleNode) { t.v.VisitHRule(hn) } @@ -54,33 +54,33 @@ // VisitDescriptionList traverses all description terms and their associated // descriptions. func (t TopDownTraverser) VisitDescriptionList(dn *DescriptionListNode) { t.v.VisitDescriptionList(dn) for _, defs := range dn.Descriptions { - t.visitInlineSlice(defs.Term) + t.VisitInlineSlice(defs.Term) for _, descr := range defs.Descriptions { t.visitDescriptionSlice(descr) } } } // VisitPara traverses the inlines of a paragraph. func (t TopDownTraverser) VisitPara(pn *ParaNode) { t.v.VisitPara(pn) - t.visitInlineSlice(pn.Inlines) + t.VisitInlineSlice(pn.Inlines) } // VisitTable traverses all cells of the header and then row-wise all cells of // the table body. func (t TopDownTraverser) VisitTable(tn *TableNode) { t.v.VisitTable(tn) for _, col := range tn.Header { - t.visitInlineSlice(col.Inlines) + t.VisitInlineSlice(col.Inlines) } for _, row := range tn.Rows { for _, col := range row { - t.visitInlineSlice(col.Inlines) + t.VisitInlineSlice(col.Inlines) } } } // VisitBLOB traverses nothing. @@ -99,38 +99,38 @@ func (t TopDownTraverser) VisitBreak(bn *BreakNode) { t.v.VisitBreak(bn) } // VisitLink traverses the link text. func (t TopDownTraverser) VisitLink(ln *LinkNode) { t.v.VisitLink(ln) - t.visitInlineSlice(ln.Inlines) + t.VisitInlineSlice(ln.Inlines) } // VisitImage traverses the image text. func (t TopDownTraverser) VisitImage(in *ImageNode) { t.v.VisitImage(in) - t.visitInlineSlice(in.Inlines) + t.VisitInlineSlice(in.Inlines) } // VisitCite traverses the cite text. func (t TopDownTraverser) VisitCite(cn *CiteNode) { t.v.VisitCite(cn) - t.visitInlineSlice(cn.Inlines) + t.VisitInlineSlice(cn.Inlines) } // VisitFootnote traverses the footnote text. func (t TopDownTraverser) VisitFootnote(fn *FootnoteNode) { t.v.VisitFootnote(fn) - t.visitInlineSlice(fn.Inlines) + t.VisitInlineSlice(fn.Inlines) } // VisitMark traverses nothing. func (t TopDownTraverser) VisitMark(mn *MarkNode) { t.v.VisitMark(mn) } // VisitFormat traverses the formatted text. func (t TopDownTraverser) VisitFormat(fn *FormatNode) { t.v.VisitFormat(fn) - t.visitInlineSlice(fn.Inlines) + t.VisitInlineSlice(fn.Inlines) } // VisitLiteral traverses nothing. func (t TopDownTraverser) VisitLiteral(ln *LiteralNode) { t.v.VisitLiteral(ln) } @@ -151,10 +151,11 @@ for _, dn := range dns { dn.Accept(t) } } -func (t TopDownTraverser) visitInlineSlice(ins InlineSlice) { +// VisitInlineSlice traverses a block slice. +func (t TopDownTraverser) VisitInlineSlice(ins InlineSlice) { for _, in := range ins { in.Accept(t) } } Index: auth/policy/place.go ================================================================== --- auth/policy/place.go +++ auth/policy/place.go @@ -16,10 +16,11 @@ "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/place" + "zettelstore.de/z/search" "zettelstore.de/z/web/session" ) // PlaceWithPolicy wraps the given place inside a policy place. func PlaceWithPolicy( @@ -91,26 +92,15 @@ func (pp *polPlace) FetchZids(ctx context.Context) (id.Set, error) { return nil, place.NewErrNotAllowed("fetch-zids", session.GetUser(ctx), id.Invalid) } -func (pp *polPlace) SelectMeta( - ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) { +func (pp *polPlace) SelectMeta(ctx context.Context, s *search.Search) ([]*meta.Meta, error) { user := session.GetUser(ctx) - f = place.EnsureFilter(f) canRead := pp.policy.CanRead - if sel := f.Select; sel != nil { - f.Select = func(m *meta.Meta) bool { - return canRead(user, m) && sel(m) - } - } else { - f.Select = func(m *meta.Meta) bool { - return canRead(user, m) - } - } - result, err := pp.place.SelectMeta(ctx, f, s) - return result, err + s = s.AddPreMatch(func(m *meta.Meta) bool { return canRead(user, m) }) + return pp.place.SelectMeta(ctx, s) } func (pp *polPlace) CanUpdateZettel(ctx context.Context, zettel domain.Zettel) bool { return pp.place.CanUpdateZettel(ctx, zettel) } Index: cmd/cmd_file.go ================================================================== --- cmd/cmd_file.go +++ cmd/cmd_file.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -11,11 +11,11 @@ package cmd import ( "flag" "fmt" - "io/ioutil" + "io" "os" "zettelstore.de/z/config/runtime" "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" @@ -38,14 +38,11 @@ Meta: meta, Content: domain.NewContent(inp.Src[inp.Pos:]), }, runtime.GetSyntax(meta), ) - enc := encoder.Create( - format, - &encoder.StringOption{Key: "lang", Value: runtime.GetLang(meta)}, - ) + enc := encoder.Create(format, &encoder.Environment{Lang: runtime.GetLang(meta)}) if enc == nil { fmt.Fprintf(os.Stderr, "Unknown format %q\n", format) return 2, nil } _, err = enc.WriteZettel(os.Stdout, z, format != "raw") @@ -57,30 +54,30 @@ return 0, nil } func getInput(args []string) (*meta.Meta, *input.Input, error) { if len(args) < 1 { - src, err := ioutil.ReadAll(os.Stdin) + src, err := io.ReadAll(os.Stdin) if err != nil { return nil, nil, err } inp := input.NewInput(string(src)) m := meta.NewFromInput(id.New(true), inp) return m, inp, nil } - src, err := ioutil.ReadFile(args[0]) + src, err := os.ReadFile(args[0]) if err != nil { return nil, nil, err } inp := input.NewInput(string(src)) m := meta.NewFromInput(id.New(true), inp) if len(args) > 1 { - src, err := ioutil.ReadFile(args[1]) + src, err := os.ReadFile(args[1]) if err != nil { return nil, nil, err } inp = input.NewInput(string(src)) } return m, inp, nil } Index: cmd/cmd_password.go ================================================================== --- cmd/cmd_password.go +++ cmd/cmd_password.go @@ -13,11 +13,11 @@ import ( "flag" "fmt" "os" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" "zettelstore.de/z/auth/cred" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" ) @@ -66,9 +66,9 @@ return 0, nil } func getPassword(prompt string) (string, error) { fmt.Fprintf(os.Stderr, "%s: ", prompt) - password, err := terminal.ReadPassword(int(os.Stdin.Fd())) + password, err := term.ReadPassword(int(os.Stdin.Fd())) fmt.Fprintln(os.Stderr) return string(password), err } Index: cmd/cmd_run.go ================================================================== --- cmd/cmd_run.go +++ cmd/cmd_run.go @@ -84,42 +84,42 @@ ucListRoles := usecase.NewListRole(pp) ucListTags := usecase.NewListTags(pp) ucZettelContext := usecase.NewZettelContext(pp) router := router.NewRouter() - router.Handle("/", webui.MakeGetRootHandler(pp)) + router.Handle("/", webui.MakeGetRootHandler(te, pp)) router.AddListRoute('a', http.MethodGet, webui.MakeGetLoginHandler(te)) router.AddListRoute('a', http.MethodPost, adapter.MakePostLoginHandler( api.MakePostLoginHandlerAPI(ucAuthenticate), webui.MakePostLoginHandlerHTML(te, ucAuthenticate))) router.AddListRoute('a', http.MethodPut, api.MakeRenewAuthHandler()) - router.AddZettelRoute('a', http.MethodGet, webui.MakeGetLogoutHandler()) + router.AddZettelRoute('a', http.MethodGet, webui.MakeGetLogoutHandler(te)) if !readonlyMode { router.AddZettelRoute('b', http.MethodGet, webui.MakeGetRenameZettelHandler( te, ucGetMeta)) router.AddZettelRoute('b', http.MethodPost, webui.MakePostRenameZettelHandler( - usecase.NewRenameZettel(pp))) + te, usecase.NewRenameZettel(pp))) router.AddZettelRoute('c', http.MethodGet, webui.MakeGetCopyZettelHandler( te, ucGetZettel, usecase.NewCopyZettel())) router.AddZettelRoute('c', http.MethodPost, webui.MakePostCreateZettelHandler( - usecase.NewCreateZettel(pp))) + te, usecase.NewCreateZettel(pp))) router.AddZettelRoute('d', http.MethodGet, webui.MakeGetDeleteZettelHandler( te, ucGetZettel)) router.AddZettelRoute('d', http.MethodPost, webui.MakePostDeleteZettelHandler( - usecase.NewDeleteZettel(pp))) + te, usecase.NewDeleteZettel(pp))) router.AddZettelRoute('e', http.MethodGet, webui.MakeEditGetZettelHandler( te, ucGetZettel)) router.AddZettelRoute('e', http.MethodPost, webui.MakeEditSetZettelHandler( - usecase.NewUpdateZettel(pp))) + te, usecase.NewUpdateZettel(pp))) router.AddZettelRoute('f', http.MethodGet, webui.MakeGetFolgeZettelHandler( te, ucGetZettel, usecase.NewFolgeZettel())) router.AddZettelRoute('f', http.MethodPost, webui.MakePostCreateZettelHandler( - usecase.NewCreateZettel(pp))) + te, usecase.NewCreateZettel(pp))) router.AddZettelRoute('g', http.MethodGet, webui.MakeGetNewZettelHandler( te, ucGetZettel, usecase.NewNewZettel())) router.AddZettelRoute('g', http.MethodPost, webui.MakePostCreateZettelHandler( - usecase.NewCreateZettel(pp))) + te, usecase.NewCreateZettel(pp))) } router.AddListRoute('f', http.MethodGet, webui.MakeSearchHandler( te, usecase.NewSearch(pp), ucGetMeta, ucGetZettel)) router.AddListRoute('h', http.MethodGet, webui.MakeListHTMLMetaHandler( te, ucListMeta, ucListRoles, ucListTags)) Index: cmd/main.go ================================================================== --- cmd/main.go +++ cmd/main.go @@ -12,11 +12,10 @@ import ( "context" "flag" "fmt" - "io/ioutil" "os" "strings" "zettelstore.de/z/config/runtime" "zettelstore.de/z/config/startup" @@ -95,11 +94,11 @@ if configFlag := fs.Lookup("c"); configFlag != nil { configFile = configFlag.Value.String() } else { configFile = defConfigfile } - content, err := ioutil.ReadFile(configFile) + content, err := os.ReadFile(configFile) if err != nil { cfg = meta.New(id.Invalid) } else { cfg = meta.NewFromInput(id.Invalid, input.NewInput(string(content))) } Index: cmd/register.go ================================================================== --- cmd/register.go +++ cmd/register.go @@ -24,7 +24,8 @@ _ "zettelstore.de/z/parser/none" // Allow to use none parser. _ "zettelstore.de/z/parser/plain" // Allow to use plain parser. _ "zettelstore.de/z/parser/zettelmark" // Allow to use zettelmark parser. _ "zettelstore.de/z/place/constplace" // Allow to use global internal place. _ "zettelstore.de/z/place/dirplace" // Allow to use directory place. + _ "zettelstore.de/z/place/fileplace" // Allow to use file place. _ "zettelstore.de/z/place/memplace" // Allow to use memory place. ) Index: cmd/zettelstore/main.go ================================================================== --- cmd/zettelstore/main.go +++ cmd/zettelstore/main.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -9,15 +9,13 @@ //----------------------------------------------------------------------------- // Package main is the starting point for the zettelstore command. package main -import ( - "zettelstore.de/z/cmd" -) +import "zettelstore.de/z/cmd" // Version variable. Will be filled by build process. var version string = "" func main() { cmd.Main("Zettelstore", version) } Index: config/runtime/runtime.go ================================================================== --- config/runtime/runtime.go +++ config/runtime/runtime.go @@ -41,13 +41,15 @@ return configStock.GetMeta(id.ConfigurationZid) } // GetDefaultTitle returns the current value of the "default-title" key. func GetDefaultTitle() string { - if config := getConfigurationMeta(); config != nil { - if title, ok := config.Get(meta.KeyDefaultTitle); ok { - return title + if configStock != nil { + if config := getConfigurationMeta(); config != nil { + if title, ok := config.Get(meta.KeyDefaultTitle); ok { + return title + } } } return "Untitled" } @@ -93,11 +95,10 @@ if config := getConfigurationMeta(); config != nil { if copyright, ok := config.Get(meta.KeyDefaultCopyright); ok { return copyright } } - // TODO: get owner } return "" } // GetDefaultLicense returns the current value of the "default-license" key. Index: docs/manual/00000000000100.zettel ================================================================== --- docs/manual/00000000000100.zettel +++ docs/manual/00000000000100.zettel @@ -4,8 +4,10 @@ syntax: none default-copyright: (c) 2020-2021 by Detlef Stern default-license: EUPL-1.2-or-later default-visibility: public footer-html:

Imprint / Privacy

+home-zettel: 00001000000000 +no-index: true site-name: Zettelstore Manual visibility: owner ADDED docs/manual/00001000000000.zettel Index: docs/manual/00001000000000.zettel ================================================================== --- /dev/null +++ docs/manual/00001000000000.zettel @@ -0,0 +1,21 @@ +id: 00001000000000 +title: Zettelstore Manual +role: manual +tags: #manual #zettelstore +syntax: zmk + +* [[Introduction|00001001000000]] +* [[Design goals|00001002000000]] +* [[Installation|00001003000000]] +* [[Configuration|00001004000000]] +* [[Structure of Zettelstore|00001005000000]] +* [[Layout of a zettel|00001006000000]] +* [[Zettelmarkup|00001007000000]] +* [[Other markup languages|00001008000000]] +* [[Security|00001010000000]] +* [[API|00001012000000]] +* [[Web user interface|00001014000000]] +* Troubleshooting +* Frequently asked questions + +Licensed under the EUPL-1.2-or-later. Index: docs/manual/00001004011200.zettel ================================================================== --- docs/manual/00001004011200.zettel +++ docs/manual/00001004011200.zettel @@ -1,18 +1,18 @@ id: 00001004011200 title: Zettelstore places +role: manual tags: #configuration #manual #zettelstore syntax: zmk -role: manual A Zettelstore must store its zettel somehow and somewhere. In most cases you want to store your zettel as files in a directory. Under certain circumstances you may want to store your zettel in other places. An example are the [[predefined zettel|00001005090000]] that come with a Zettelstore. They are stored within the software itself. -In another situation you may want to store your zettel volatilely, e.g. if you want to provide a sandbox for experimenting. +In another situation you may want to store your zettel volatile, e.g. if you want to provide a sandbox for experimenting. To cope with these (and more) situations, you configure Zettelstore to use one or more places. This is done via the ''place-X-uri'' keys of the [[start-up configuration|00001004010000#place-X-uri]] (X is a number). Places are specified using special [[URIs|https://en.wikipedia.org/wiki/Uniform_Resource_Identifier]], somehow similar to web addresses. @@ -24,15 +24,18 @@ Although it is possible to use relative file paths, such as ''./zettel'' (→ URI is ''dir:\//.zettel''), it is preferable to use absolute file paths, e.g. ''/home/user/zettel''. The directory must exist before starting the Zettelstore[^There is one exception: when Zettelstore is [[started without any parameter|00001004050000]], e.g. via double-clicking its icon, an directory called ''./zettel'' will be created.]. It is possible to [[configure|00001004011400]] a directory place. +; ''file:FILE.zip'' oder ''file:/\//path/to/file.zip'' +: Specifies a ZIP file which contains files that store zettel. + You can create such a ZIP file, if you zip a directory full of zettel files. + + This place is always read-only. ; ''mem:'' : Stores all its zettel in volatile memory. If you stop the Zettelstore, all changes are lost. -; ''const:'' -: Is a place of predefined, essential zettel. All places that you configure via the ''store-X-uri'' keys form a chain of places. If a zettel should be retrieved, a search starts in the place specified with the ''place-1-uri'' key, then ''place-2-uri'' and so on. If a zettel is created or changed, it is always stored in the place specified with the ''place-1-uri'' key. This allows to overwrite zettel from other places, e.g. the predefined zettel. Index: docs/manual/00001006020000.zettel ================================================================== --- docs/manual/00001006020000.zettel +++ docs/manual/00001006020000.zettel @@ -44,12 +44,12 @@ : Date and time when a zettel was modified through Zettelstore. If you edit a zettel with an editor software outside Zettelstore, you should set it manually to an appropriate value. This is a computed value. There is no need to set it via Zettelstore. -; [!new-role]''new-role'' -: Used in a template zettel to specify the ''role'' of the new zettel. +; [!no-index]''no-index'' +: If set to true, the zettel will not be indexed and therefore not be found in full-text searches. ; [!precursor]''precursor'' : References zettel for which this zettel is a ""Folgezettel"" / follow-up zettel. Basically the inverse of key [[''folge''|#folge]]. ; [!published]''published'' : This property contains the timestamp of the mast modification / creation of the zettel. Index: docs/manual/00001006034000.zettel ================================================================== --- docs/manual/00001006034000.zettel +++ docs/manual/00001006034000.zettel @@ -11,9 +11,16 @@ === Allowed values Every tag must must begin with the number sign character (""''#''"", ''U+0023''), followed by at least one printable character. Tags are separated by space characters. === Match operator -A value matches a tag set value, if the first value is equal to at least one tag in the tag set. +It depends of the first character of a search string how it is matched against a tag set value: + +* If the first character of the search string is a number sign character, + it must exactly match one of the values of a tag. +* In other cases, the search string must be the prefix of at least one tag. + +Conpectually, all number sign characters are removed at the beginning of the search string +and of all tags. === Sorting Sorting is done by comparing the [[String|00001006033500]] values. Index: docs/manual/00001006050000.zettel ================================================================== --- docs/manual/00001006050000.zettel +++ docs/manual/00001006050000.zettel @@ -19,9 +19,9 @@ However, the Zettelstore software just checks for exactly 14 digits. Anybody is free to assign a ""non-timestamp"" identifier to a zettel, e.g. with a month part of ""35"" or with ""99"" as the last two digits. In fact, all identifiers of zettel initially provided by an empty Zettelstore begin with ""000000"", except the home zettel ''00010000000000''. -The identifiers of zettel if this manual have be chosen to begin with ""000010"". +Zettel identifier of this manual have be chosen to begin with ""000010"". A zettel can have any identifier that contains 14 digits and that is not in use by another zettel managed by the same Zettelstore. Index: docs/manual/00001010070200.zettel ================================================================== --- docs/manual/00001010070200.zettel +++ docs/manual/00001010070200.zettel @@ -35,7 +35,7 @@ The other zettel is the zettel containing the [[version|00000000000001]] of the Zettelstore. Please note: if authentication is not enabled, every user has the same rights as the owner of a Zettelstore. This is also true, if the Zettelstore runs additionally in [[read-only mode|00001004010000#read-only-mode]]. In this case, the [[runtime configuration zettel|00001004020000]] is shown (its visibility is ""owner""). -The [[start-up configuration|00001004010000]] is not shown, because the associated computed zettel with identifier ''00000000000099'' is stored with the visibility ""expert"". +The [[start-up configuration|00001004010000]] is not shown, because the associated computed zettel with identifier ''00000000000098'' is stored with the visibility ""expert"". If you want to show such a zettel, you must set ''expert-mode'' to true. Index: docs/manual/00001012051800.zettel ================================================================== --- docs/manual/00001012051800.zettel +++ docs/manual/00001012051800.zettel @@ -17,10 +17,23 @@ For example, if you want to retrieve all zettel that contain the string ""API"" in its title, your request will be: ```sh # curl 'http://127.0.0.1:23123/z?title=API' {"list":[{"id":"00001012921000","url":"/z/00001012921000","meta":{"title":"API: JSON structure of an access token","tags":"#api #manual #reference #zettelstore","syntax":"zmk","role":"manual"}},{"id":"00001012920500","url":"/z/00001012920500","meta":{"title":"Formats available by the API","tags":"#api #manual #reference #zettelstore","syntax":"zmk","role":"manual"}},{"id":"00001012920000","url":"/z/00001012920000","meta":{"title":"Endpoints used by the API","tags":"#api #manual #reference #zettelstore","syntax":"zmk","role":"manual"}}, ... ``` + +However, if you want all zettel that does //not// match a given value, you must prefix the value with the exclamation mark character (""!"", ''U+0021''). +For example, if you want to retrieve all zettel that do not contain the string ""API"" in their title, your request will be: +```sh +# curl 'http://127.0.0.1:23123/z?title=!API' +{"list":[{"id":"00010000000000","url":"/z/00010000000000","meta":{"back":"00001003000000 00001005090000","backward":"00001003000000 00001005090000","copyright":"(c) 2020-2021 by Detlef Stern ","forward":"00000000000001 00000000000003 00000000000096 00000000000098 00000000000100","lang":"en","license":"EUPL-1.2-or-later","role":"zettel","syntax":"zmk","title":"Home"}},{"id":"00001014000000","url":"/z/00001014000000","meta":{"back":"00001000000000 00001004020000 00001012920510","backward":"00001000000000 00001004020000 00001012000000 00001012920510","copyright":"(c) 2020-2021 by Detlef Stern ","forward":"00001012000000","lang":"en","license":"EUPL-1.2-or-later","published":"00001014000000","role":"manual","syntax":"zmk","tags":"#manual #webui #zettelstore","title":"Web user interface"}}, +... +``` +The empty query parameter values matches all zettel that contain the given metadata key. +Similar, if you specify just the exclamation mark character as a query parameter value, only those zettel match that does //not// contain the given metadata key. +For example ``curl 'http://localhost:23123/z?back=!&backward='`` returns all zettel that are reachable via other zettel, but also references these zettel. + +=== Output only specific parts of a zettel If you are just interested in the zettel identifier, you should add the ""''_part''"" query parameter: ```sh # curl 'http://127.0.0.1:23123/z?title=API&_part=id' {"list":[{"id":"00001012921000","url":"/z/00001012921000"},{"id":"00001012920500","url":"/z/00001012920500"},{"id":"00001012920000","url":"/z/00001012920000"},{"id":"00001012051800","url":"/z/00001012051800"},{"id":"00001012051600","url":"/z/00001012051600"},{"id":"00001012051400","url":"/z/00001012051400"},{"id":"00001012051200","url":"/z/00001012051200"},{"id":"00001012050600","url":"/z/00001012050600"},{"id":"00001012050400","url":"/z/00001012050400"},{"id":"00001012050200","url":"/z/00001012050200"},{"id":"00001012000000","url":"/z/00001012000000"}]} ``` @@ -46,12 +59,12 @@ # curl 'http://192.168.17.7:23121/z?title=API&_part=id&_sort=id&_limit=2&_offset=1' {"list":[{"id":"00001012050200","url":"/z/00001012050200"},{"id":"00001012050400","url":"/z/00001012050400"}]} ``` === General filter -The query parameter ""''_s''"" allows to provide a string, which will be searched for in all metadata. -While searching, the [[type|00001006030000]] of each metadata key will be respected. +The query parameter ""''_s''"" allows to provide a string for a full-text search of all zettel. +The search string will be normalized according to Unicode NKFD, ignoring everything except letters and numbers. You are allowed to specify this query parameter more than once. All results will be intersected, i.e. a zettel will be included into the list if both of the provided values match. This parameter loosely resembles the search box of the web user interface. Index: docs/manual/00001012054000.zettel ================================================================== --- docs/manual/00001012054000.zettel +++ docs/manual/00001012054000.zettel @@ -3,11 +3,11 @@ role: manual tags: #api #manual #zettelstore syntax: zmk Some zettel act as a ""table of contents"" for other zettel. -The [[Home zettel|00010000000000]] of this manual is one example, the [[general API description|00001012000000]] is another. +The [[initial zettel|00001000000000]] of this manual is one example, the [[general API description|00001012000000]] is another. Every zettel with a certain internal structure can act as the ""table of contents"" for others. What is a ""table of contents""? Basically, it is just a list of references to other zettel. @@ -19,18 +19,18 @@ Following references to zettel within such an list item are ignored. To retrieve the zettel order of an existing zettel, use the [[endpoint|00001012920000]] ''/o/{ID}''. ```` -# curl http://127.0.0.1:23123/o/00010000000000 -{"id":"00010000000000","url":"/z/00010000000000","meta":{...},"list":[{"id":"00001001000000","url":"/z/00001001000000","meta":{...}},{"id":"00001002000000","url":"/z/00001002000000","meta":{...}},{"id":"00001003000000","url":"/z/00001003000000","meta":{...}},{"id":"00001004000000","url":"/z/00001004000000","meta":{...}},...,{"id":"00001014000000","url":"/z/00001014000000","meta":{...}}]} +# curl http://127.0.0.1:23123/o/00001000000000 +{"id":"00001000000000","url":"/z/00001000000000","meta":{...},"list":[{"id":"00001001000000","url":"/z/00001001000000","meta":{...}},{"id":"00001002000000","url":"/z/00001002000000","meta":{...}},{"id":"00001003000000","url":"/z/00001003000000","meta":{...}},{"id":"00001004000000","url":"/z/00001004000000","meta":{...}},...,{"id":"00001014000000","url":"/z/00001014000000","meta":{...}}]} ```` Formatted, this translates into:[^Metadata (key ''meta'') are hidden to make the overall structure easier to read.] ````json { - "id": "00010000000000", - "url": "/z/00010000000000", + "id": "00001000000000", + "url": "/z/00001000000000", "order": [ { "id": "00001001000000", "url": "/z/00001001000000", "meta": {...} DELETED docs/manual/00010000000000.zettel Index: docs/manual/00010000000000.zettel ================================================================== --- docs/manual/00010000000000.zettel +++ /dev/null @@ -1,21 +0,0 @@ -id: 00001000000000 -title: Zettelstore Manual -role: manual -tags: #manual #zettelstore -syntax: zmk - -* [[Introduction|00001001000000]] -* [[Design goals|00001002000000]] -* [[Installation|00001003000000]] -* [[Configuration|00001004000000]] -* [[Structure of Zettelstore|00001005000000]] -* [[Layout of a zettel|00001006000000]] -* [[Zettelmarkup|00001007000000]] -* [[Other markup languages|00001008000000]] -* [[Security|00001010000000]] -* [[API|00001012000000]] -* [[Web user interface|00001014000000]] -* Troubleshooting -* Frequently asked questions - -Licensed under the EUPL-1.2-or-later. ADDED docs/readmezip.txt Index: docs/readmezip.txt ================================================================== --- /dev/null +++ docs/readmezip.txt @@ -0,0 +1,21 @@ +Zettelstore +=========== + +Zettelstore is a software that collects and relates your notes ("zettel") to +represent and enhance your knowledge. It helps with many tasks of personal +knowledge management by explicitly supporting the Zettelkasten method (see: +https://en.wikipedia.org/wiki/Zettelkasten). The method is based on creating +many individual notes, each with one idea or information, that are related to +each other. Since knowledge is typically build up gradually, one major focus is +a long-term store of these notes, hence the name "Zettelstore". + +To get an impression, take a look at the manual at +https://zettelstore.de/manual/. It is a live example of the zettelstore +software, running in read-only mode. You can download it separately and it is +possible to make it directly available for your local Zettelstore. + +The software, including the manual, is licensed under the European Union Public +License 1.2 (or later). See the separate file LICENSE.txt. + +To get in contact with the developer, send an email to ds@zettelstore.de or +follow Zettelstore on Twitter: https://twitter.com/zettelstore. Index: domain/content.go ================================================================== --- domain/content.go +++ domain/content.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -9,13 +9,11 @@ //----------------------------------------------------------------------------- // Package domain provides domain specific types, constants, and functions. package domain -import ( - "unicode/utf8" -) +import "unicode/utf8" // Content is just the uninterpreted content of a zettel. type Content string // NewContent creates a new content from a string. Index: domain/id/id.go ================================================================== --- domain/id/id.go +++ domain/id/id.go @@ -33,18 +33,19 @@ // WebUI HTML templates are in the range 10000..19999 BaseTemplateZid = Zid(10100) LoginTemplateZid = Zid(10200) ListTemplateZid = Zid(10300) - DetailTemplateZid = Zid(10401) + ZettelTemplateZid = Zid(10401) InfoTemplateZid = Zid(10402) FormTemplateZid = Zid(10403) RenameTemplateZid = Zid(10404) DeleteTemplateZid = Zid(10405) ContextTemplateZid = Zid(10406) RolesTemplateZid = Zid(10500) TagsTemplateZid = Zid(10600) + ErrorTemplateZid = Zid(10700) // WebUI CSS pages are in the range 20000..29999 BaseCSSZid = Zid(20001) // WebUI JS pages are in the range 30000..39999 Index: domain/id/set.go ================================================================== --- domain/id/set.go +++ domain/id/set.go @@ -42,12 +42,12 @@ result[zid] = true } return result } -// Sort returns the set as a sorted slice of zettel identifier. -func (s Set) Sort() Slice { +// Sorted returns the set as a sorted slice of zettel identifier. +func (s Set) Sorted() Slice { if l := len(s); l > 0 { result := make(Slice, 0, l) for zid := range s { result = append(result, zid) } @@ -54,5 +54,25 @@ result.Sort() return result } return nil } + +// Intersect removes all zettel identifier that are not in the other set. +// Both sets can be modified by this method. One of them is the set returned. +// It contains the intersection of both. +func (s Set) Intersect(other Set) Set { + if len(s) > len(other) { + s, other = other, s + } + for zid, inSet := range s { + if !inSet { + delete(s, zid) + continue + } + otherInSet, otherOk := other[zid] + if !otherInSet || !otherOk { + delete(s, zid) + } + } + return s +} ADDED domain/id/set_test.go Index: domain/id/set_test.go ================================================================== --- /dev/null +++ domain/id/set_test.go @@ -0,0 +1,63 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package id provides domain specific types, constants, and functions about +// zettel identifier. +package id_test + +import ( + "testing" + + "zettelstore.de/z/domain/id" +) + +func TestSetSorted(t *testing.T) { + testcases := []struct { + set id.Set + exp id.Slice + }{ + {nil, nil}, + {id.NewSet(), nil}, + {id.NewSet(9, 4, 6, 1, 7), id.Slice{1, 4, 6, 7, 9}}, + } + for i, tc := range testcases { + got := tc.set.Sorted() + if !got.Equal(tc.exp) { + t.Errorf("%d: %v.Sorted() should be %v, but got %v", i, tc.set, tc.exp, got) + } + } +} + +func TestSetIntersection(t *testing.T) { + testcases := []struct { + s1, s2 id.Set + exp id.Slice + }{ + {nil, nil, nil}, + {id.NewSet(), nil, nil}, + {id.NewSet(), id.NewSet(), nil}, + {id.NewSet(1), nil, nil}, + {id.NewSet(1), id.NewSet(), nil}, + {id.NewSet(1), id.NewSet(2), nil}, + {id.NewSet(1), id.NewSet(1), id.Slice{1}}, + } + for i, tc := range testcases { + sl1 := tc.s1.Sorted() + sl2 := tc.s2.Sorted() + got := tc.s1.Intersect(tc.s2).Sorted() + if !got.Equal(tc.exp) { + t.Errorf("%d: %v.Intersect(%v) should be %v, but got %v", i, sl1, sl2, tc.exp, got) + } + got = id.NewSet(sl2...).Intersect(id.NewSet(sl1...)).Sorted() + if !got.Equal(tc.exp) { + t.Errorf("%d: %v.Intersect(%v) should be %v, but got %v", i, sl2, sl1, tc.exp, got) + } + } +} Index: domain/id/slice.go ================================================================== --- domain/id/slice.go +++ domain/id/slice.go @@ -34,10 +34,27 @@ } result := make(Slice, len(zs)) copy(result, zs) return result } + +// Equal reports whether zs and other are the same length and contain the samle zettel +// identifier. A nil argument is equivalent to an empty slice. +func (zs Slice) Equal(other Slice) bool { + if len(zs) != len(other) { + return false + } + if len(zs) == 0 { + return true + } + for i, e := range zs { + if e != other[i] { + return false + } + } + return true +} func (zs Slice) String() string { if len(zs) == 0 { return "" } Index: domain/id/slice_test.go ================================================================== --- domain/id/slice_test.go +++ domain/id/slice_test.go @@ -16,15 +16,16 @@ "testing" "zettelstore.de/z/domain/id" ) -func TestSort(t *testing.T) { +func TestSliceSort(t *testing.T) { zs := id.Slice{9, 4, 6, 1, 7} zs.Sort() - if zs[0] != 1 || zs[1] != 4 || zs[2] != 6 || zs[3] != 7 || zs[4] != 9 { - t.Errorf("Slice.Sort did not work. Expected %v, got %v", id.Slice{1, 4, 6, 7, 9}, zs) + exp := id.Slice{1, 4, 6, 7, 9} + if !zs.Equal(exp) { + t.Errorf("Slice.Sort did not work. Expected %v, got %v", exp, zs) } } func TestCopy(t *testing.T) { var orig id.Slice @@ -32,15 +33,41 @@ if got != nil { t.Errorf("Nil copy resulted in %v", got) } orig = id.Slice{9, 4, 6, 1, 7} got = orig.Copy() - if len(got) != len(orig) || got[0] != 9 || got[1] != 4 || got[2] != 6 || got[3] != 1 || got[4] != 7 { + if !orig.Equal(got) { t.Errorf("Slice.Copy did not work. Expected %v, got %v", orig, got) } } -func TestString(t *testing.T) { + +func TestSliceEqual(t *testing.T) { + testcases := []struct { + s1, s2 id.Slice + exp bool + }{ + {nil, nil, true}, + {nil, id.Slice{}, true}, + {nil, id.Slice{1}, false}, + {id.Slice{1}, id.Slice{1}, true}, + {id.Slice{1}, id.Slice{2}, false}, + {id.Slice{1, 2}, id.Slice{2, 1}, false}, + {id.Slice{1, 2}, id.Slice{1, 2}, true}, + } + for i, tc := range testcases { + got := tc.s1.Equal(tc.s2) + if got != tc.exp { + t.Errorf("%d/%v.Equal(%v)==%v, but got %v", i, tc.s1, tc.s2, tc.exp, got) + } + got = tc.s2.Equal(tc.s1) + if got != tc.exp { + t.Errorf("%d/%v.Equal(%v)==%v, but got %v", i, tc.s2, tc.s1, tc.exp, got) + } + } +} + +func TestSliceString(t *testing.T) { testcases := []struct { in id.Slice exp string }{ {nil, ""}, Index: domain/meta/meta.go ================================================================== --- domain/meta/meta.go +++ domain/meta/meta.go @@ -134,10 +134,11 @@ KeyLang = registerKey("lang", TypeWord, usageUser, "") KeyLicense = registerKey("license", TypeEmpty, usageUser, "") KeyListPageSize = registerKey("list-page-size", TypeNumber, usageUser, "") KeyMarkerExternal = registerKey("marker-external", TypeEmpty, usageUser, "") KeyModified = registerKey("modified", TypeTimestamp, usageComputed, "") + KeyNoIndex = registerKey("no-index", TypeBool, usageUser, "") KeyPrecursor = registerKey("precursor", TypeIDSet, usageUser, KeyFolge) KeyPublished = registerKey("published", TypeTimestamp, usageProperty, "") KeyReadOnly = registerKey("read-only", TypeWord, usageUser, "") KeySiteName = registerKey("site-name", TypeString, usageUser, "") KeyURL = registerKey("url", TypeURL, usageUser, "") Index: domain/meta/meta_test.go ================================================================== --- domain/meta/meta_test.go +++ domain/meta/meta_test.go @@ -18,24 +18,10 @@ "zettelstore.de/z/domain/id" ) const testID = id.Zid(98765432101234) -func newMeta(title string, tags []string, syntax string) *Meta { - m := New(testID) - if title != "" { - m.Set(KeyTitle, title) - } - if tags != nil { - m.Set(KeyTags, strings.Join(tags, " ")) - } - if syntax != "" { - m.Set(KeySyntax, syntax) - } - return m -} - func TestKeyIsValid(t *testing.T) { validKeys := []string{"0", "a", "0-", "title", "title-----", strings.Repeat("r", 255)} for _, key := range validKeys { if !KeyIsValid(key) { t.Errorf("Key %q wrongly identified as invalid key", key) ADDED encoder/encfun/encfun.go Index: encoder/encfun/encfun.go ================================================================== --- /dev/null +++ encoder/encfun/encfun.go @@ -0,0 +1,37 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package encfun provides some helper function to work with encodings. +package encfun + +import ( + "strings" + + "zettelstore.de/z/ast" + "zettelstore.de/z/domain/meta" + "zettelstore.de/z/encoder" + "zettelstore.de/z/parser" +) + +// MetaAsInlineSlice returns the value of the given metadata key as an inlince slice. +func MetaAsInlineSlice(m *meta.Meta, key string) ast.InlineSlice { + return parser.ParseMetadata(m.GetDefault(key, "")) +} + +// MetaAsText returns the value of given metadata as text. +func MetaAsText(m *meta.Meta, key string) string { + textEncoder := encoder.Create("text", nil) + var sb strings.Builder + _, err := textEncoder.WriteInlines(&sb, MetaAsInlineSlice(m, key)) + if err == nil { + return sb.String() + } + return "" +} Index: encoder/encoder.go ================================================================== --- encoder/encoder.go +++ encoder/encoder.go @@ -22,12 +22,10 @@ "zettelstore.de/z/domain/meta" ) // Encoder is an interface that allows to encode different parts of a zettel. type Encoder interface { - SetOption(Option) - WriteZettel(io.Writer, *ast.ZettelNode, bool) (int, error) WriteMeta(io.Writer, *meta.Meta) (int, error) WriteContent(io.Writer, *ast.ZettelNode) (int, error) WriteBlocks(io.Writer, ast.BlockSlice) (int, error) WriteInlines(io.Writer, ast.InlineSlice) (int, error) @@ -40,30 +38,21 @@ ErrNoWriteContent = errors.New("method WriteContent is not implemented") ErrNoWriteBlocks = errors.New("method WriteBlocks is not implemented") ErrNoWriteInlines = errors.New("method WriteInlines is not implemented") ) -// Option allows to configure an encoder -type Option interface { - Name() string -} - // Create builds a new encoder with the given options. -func Create(format string, options ...Option) Encoder { +func Create(format string, env *Environment) Encoder { if info, ok := registry[format]; ok { - enc := info.Create() - for _, opt := range options { - enc.SetOption(opt) - } - return enc + return info.Create(env) } return nil } // Info stores some data about an encoder. type Info struct { - Create func() Encoder + Create func(*Environment) Encoder Default bool } var registry = map[string]Info{} var defFormat string ADDED encoder/env.go Index: encoder/env.go ================================================================== --- /dev/null +++ encoder/env.go @@ -0,0 +1,112 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package encoder provides a generic interface to encode the abstract syntax +// tree into some text form. +package encoder + +import "zettelstore.de/z/ast" + +// Environment specifies all data and functions that affects encoding. +type Environment struct { + // Important for many encoder. + LinkAdapter func(*ast.LinkNode) ast.InlineNode + ImageAdapter func(*ast.ImageNode) ast.InlineNode + CiteAdapter func(*ast.CiteNode) ast.InlineNode + + // Important for HTML encoder + Lang string // default language + Interactive bool // Encoded data will be placed in interactive content + Xhtml bool // use XHTML syntax instead of HTML syntax + MarkerExternal string // Marker after link to (external) material. + NewWindow bool // open link in new window + IgnoreMeta map[string]bool + footnotes []*ast.FootnoteNode // Stores footnotes detected while encoding +} + +// AdaptLink helps to call the link adapter. +func (env *Environment) AdaptLink(ln *ast.LinkNode) (*ast.LinkNode, ast.InlineNode) { + if env == nil || env.LinkAdapter == nil { + return ln, nil + } + n := env.LinkAdapter(ln) + if n == nil { + return ln, nil + } + if ln2, ok := n.(*ast.LinkNode); ok { + return ln2, nil + } + return nil, n +} + +// AdaptImage helps to call the link adapter. +func (env *Environment) AdaptImage(in *ast.ImageNode) (*ast.ImageNode, ast.InlineNode) { + if env == nil || env.ImageAdapter == nil { + return in, nil + } + n := env.ImageAdapter(in) + if n == nil { + return in, nil + } + if in2, ok := n.(*ast.ImageNode); ok { + return in2, nil + } + return nil, n +} + +// AdaptCite helps to call the link adapter. +func (env *Environment) AdaptCite(cn *ast.CiteNode) (*ast.CiteNode, ast.InlineNode) { + if env == nil || env.CiteAdapter == nil { + return cn, nil + } + n := env.CiteAdapter(cn) + if n == nil { + return cn, nil + } + if cn2, ok := n.(*ast.CiteNode); ok { + return cn2, nil + } + return nil, n +} + +// IsInteractive returns true, if Interactive is enabled and currently embedded +// interactive encoding will take place. +func (env *Environment) IsInteractive(inInteractive bool) bool { + return inInteractive && env != nil && env.Interactive +} + +// IsXHTML return true, if XHTML is enabled. +func (env *Environment) IsXHTML() bool { + return env != nil && env.Xhtml +} + +// HasNewWindow retruns true, if a new browser windows should be opened. +func (env *Environment) HasNewWindow() bool { + return env != nil && env.NewWindow +} + +// AddFootnote adds a footnote node to the environment and returns the number of that footnote. +func (env *Environment) AddFootnote(fn *ast.FootnoteNode) int { + if env == nil { + return 0 + } + env.footnotes = append(env.footnotes, fn) + return len(env.footnotes) +} + +// GetCleanFootnotes returns the list of remembered footnote and forgets about them. +func (env *Environment) GetCleanFootnotes() []*ast.FootnoteNode { + if env == nil { + return nil + } + result := env.footnotes + env.footnotes = nil + return result +} Index: encoder/htmlenc/block.go ================================================================== --- encoder/htmlenc/block.go +++ encoder/htmlenc/block.go @@ -140,11 +140,11 @@ // VisitHRule writes HTML code for a horizontal rule:
. func (v *visitor) VisitHRule(hn *ast.HRuleNode) { v.b.WriteString("\n") } else { v.b.WriteString(">\n") } } Index: encoder/htmlenc/htmlenc.go ================================================================== --- encoder/htmlenc/htmlenc.go +++ encoder/htmlenc/htmlenc.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -10,91 +10,47 @@ // Package htmlenc encodes the abstract syntax tree into HTML5. package htmlenc import ( - "fmt" "io" "strings" "zettelstore.de/z/ast" "zettelstore.de/z/domain/meta" "zettelstore.de/z/encoder" + "zettelstore.de/z/encoder/encfun" + "zettelstore.de/z/parser" ) func init() { encoder.Register("html", encoder.Info{ - Create: func() encoder.Encoder { return &htmlEncoder{} }, + Create: func(env *encoder.Environment) encoder.Encoder { return &htmlEncoder{env: env} }, }) } type htmlEncoder struct { - lang string // default language - xhtml bool // use XHTML syntax instead of HTML syntax - markerExternal string // Marker after link to (external) material. - newWindow bool // open link in new window - adaptLink func(*ast.LinkNode) ast.InlineNode - adaptImage func(*ast.ImageNode) ast.InlineNode - adaptCite func(*ast.CiteNode) ast.InlineNode - ignoreMeta map[string]bool - footnotes []*ast.FootnoteNode -} - -func (he *htmlEncoder) SetOption(option encoder.Option) { - switch opt := option.(type) { - case *encoder.StringOption: - switch opt.Key { - case "lang": - he.lang = opt.Value - case meta.KeyMarkerExternal: - he.markerExternal = opt.Value - } - case *encoder.BoolOption: - switch opt.Key { - case "newwindow": - he.newWindow = opt.Value - case "xhtml": - he.xhtml = opt.Value - } - case *encoder.StringsOption: - if opt.Key == "no-meta" { - he.ignoreMeta = make(map[string]bool, len(opt.Value)) - for _, v := range opt.Value { - he.ignoreMeta[v] = true - } - } - case *encoder.AdaptLinkOption: - he.adaptLink = opt.Adapter - case *encoder.AdaptImageOption: - he.adaptImage = opt.Adapter - case *encoder.AdaptCiteOption: - he.adaptCite = opt.Adapter - default: - var name string - if option != nil { - name = option.Name() - } - fmt.Println("HESO", option, name) - } + env *encoder.Environment } // WriteZettel encodes a full zettel as HTML5. -func (he *htmlEncoder) WriteZettel( - w io.Writer, zn *ast.ZettelNode, inhMeta bool) (int, error) { +func (he *htmlEncoder) WriteZettel(w io.Writer, zn *ast.ZettelNode, inhMeta bool) (int, error) { v := newVisitor(he, w) - if !he.xhtml { + if !he.env.IsXHTML() { v.b.WriteString("\n") } - v.b.WriteStrings("\n\n\n") - textEnc := encoder.Create("text") - var sb strings.Builder - textEnc.WriteInlines(&sb, zn.Title) - v.b.WriteStrings("", sb.String(), "") + if env := he.env; env != nil && env.Lang == "" { + v.b.WriteStrings("\n") + } else { + v.b.WriteStrings("") + } + v.b.WriteString("\n\n\n") + v.b.WriteStrings("", encfun.MetaAsText(zn.InhMeta, meta.KeyTitle), "") if inhMeta { - v.acceptMeta(zn.InhMeta, false) + v.acceptMeta(zn.InhMeta) } else { - v.acceptMeta(zn.Zettel.Meta, false) + v.acceptMeta(zn.Meta) } v.b.WriteString("\n\n\n") v.acceptBlockSlice(zn.Ast) v.writeEndnotes() v.b.WriteString("\n") @@ -103,11 +59,23 @@ } // WriteMeta encodes meta data as HTML5. func (he *htmlEncoder) WriteMeta(w io.Writer, m *meta.Meta) (int, error) { v := newVisitor(he, w) - v.acceptMeta(m, true) + + // Write title + if title, ok := m.Get(meta.KeyTitle); ok { + textEnc := encoder.Create("text", nil) + var sb strings.Builder + textEnc.WriteInlines(&sb, parser.ParseMetadata(title)) + v.b.WriteStrings("") + } + + // Write other metadata + v.acceptMeta(m) length, err := v.b.Flush() return length, err } func (he *htmlEncoder) WriteContent(w io.Writer, zn *ast.ZettelNode) (int, error) { @@ -124,9 +92,12 @@ } // WriteInlines writes an inline slice to the writer func (he *htmlEncoder) WriteInlines(w io.Writer, is ast.InlineSlice) (int, error) { v := newVisitor(he, w) + if env := he.env; env != nil { + v.inInteractive = env.Interactive + } v.acceptInlineSlice(is) length, err := v.b.Flush() return length, err } Index: encoder/htmlenc/inline.go ================================================================== --- encoder/htmlenc/inline.go +++ encoder/htmlenc/inline.go @@ -32,21 +32,21 @@ v.b.WriteString("") } // VisitSpace emits a white space. func (v *visitor) VisitSpace(sn *ast.SpaceNode) { - if v.inVerse || v.xhtml { + if v.inVerse || v.env.IsXHTML() { v.b.WriteString(sn.Lexeme) } else { v.b.WriteByte(' ') } } // VisitBreak writes HTML code for line breaks. func (v *visitor) VisitBreak(bn *ast.BreakNode) { if bn.Hard { - if v.xhtml { + if v.env.IsXHTML() { v.b.WriteString("
\n") } else { v.b.WriteString("
\n") } } else { @@ -54,17 +54,14 @@ } } // VisitLink writes HTML code for links. func (v *visitor) VisitLink(ln *ast.LinkNode) { - if adapt := v.enc.adaptLink; adapt != nil { - n := adapt(ln) - var ok bool - if ln, ok = n.(*ast.LinkNode); !ok { - n.Accept(v) - return - } + ln, n := v.env.AdaptLink(ln) + if n != nil { + n.Accept(v) + return } v.lang.push(ln.Attrs) defer v.lang.pop() switch ln.Ref.State { @@ -76,47 +73,57 @@ attrs = attrs.Set("title", "Zettel not found") // l10n v.writeAHref(ln.Ref, attrs, ln.Inlines) case ast.RefStateExternal: attrs := ln.Attrs.Clone() attrs = attrs.Set("class", "zs-external") - if v.enc.newWindow { + if v.env.HasNewWindow() { attrs = attrs.Set("target", "_blank").Set("rel", "noopener noreferrer") } v.writeAHref(ln.Ref, attrs, ln.Inlines) - v.b.WriteString(v.enc.markerExternal) + if v.env != nil { + v.b.WriteString(v.env.MarkerExternal) + } default: + if v.env.IsInteractive(v.inInteractive) { + v.writeSpan(ln.Inlines, ln.Attrs) + return + } v.b.WriteString("') + v.inInteractive = true v.acceptInlineSlice(ln.Inlines) + v.inInteractive = false v.b.WriteString("") } } func (v *visitor) writeAHref(ref *ast.Reference, attrs *ast.Attributes, ins ast.InlineSlice) { + if v.env.IsInteractive(v.inInteractive) { + v.writeSpan(ins, attrs) + return + } v.b.WriteString("') + v.inInteractive = true v.acceptInlineSlice(ins) + v.inInteractive = false v.b.WriteString("") } // VisitImage writes HTML code for images. func (v *visitor) VisitImage(in *ast.ImageNode) { - if adapt := v.enc.adaptImage; adapt != nil { - n := adapt(in) - var ok bool - if in, ok = n.(*ast.ImageNode); !ok { - n.Accept(v) - return - } - } - + in, n := v.env.AdaptImage(in) + if n != nil { + n.Accept(v) + return + } v.lang.push(in.Attrs) defer v.lang.pop() if in.Ref == nil { v.b.WriteString("\"")") } else { v.b.WriteByte('>') } } // VisitCite writes code for citations. func (v *visitor) VisitCite(cn *ast.CiteNode) { - if adapt := v.enc.adaptCite; adapt != nil { - n := adapt(cn) - if n != cn { - n.Accept(v) - return - } - } - + cn, n := v.env.AdaptCite(cn) + if n != nil { + n.Accept(v) + return + } + if cn == nil { + return + } v.lang.push(cn.Attrs) defer v.lang.pop() - - if cn != nil { - v.b.WriteString(cn.Key) - if len(cn.Inlines) > 0 { - v.b.WriteString(", ") - v.acceptInlineSlice(cn.Inlines) - } + v.b.WriteString(cn.Key) + if len(cn.Inlines) > 0 { + v.b.WriteString(", ") + v.acceptInlineSlice(cn.Inlines) } } // VisitFootnote write HTML code for a footnote. func (v *visitor) VisitFootnote(fn *ast.FootnoteNode) { v.lang.push(fn.Attrs) defer v.lang.pop() + if v.env.IsInteractive(v.inInteractive) { + return + } - v.enc.footnotes = append(v.enc.footnotes, fn) - n := strconv.Itoa(len(v.enc.footnotes)) + n := strconv.Itoa(v.env.AddFootnote(fn)) v.b.WriteStrings("", n, "") // TODO: what to do with Attrs? } // VisitMark writes HTML code to mark a position. func (v *visitor) VisitMark(mn *ast.MarkNode) { + if v.env.IsInteractive(v.inInteractive) { + return + } if len(mn.Text) > 0 { v.b.WriteStrings("") } } @@ -214,12 +223,12 @@ case ast.FormatQuotation: code = "q" case ast.FormatSmall: code = "small" case ast.FormatSpan: - code = "span" - attrs = processSpanAttributes(attrs) + v.writeSpan(fn.Inlines, processSpanAttributes(attrs)) + return case ast.FormatMonospace: code = "span" attrs = attrs.Set("style", "font-family:monospace") case ast.FormatQuote: v.visitQuotes(fn) @@ -230,10 +239,19 @@ v.b.WriteStrings("<", code) v.visitAttributes(attrs) v.b.WriteByte('>') v.acceptInlineSlice(fn.Inlines) v.b.WriteStrings("") +} + +func (v *visitor) writeSpan(ins ast.InlineSlice, attrs *ast.Attributes) { + v.b.WriteString("') + v.acceptInlineSlice(ins) + v.b.WriteString("") + } var langQuotes = map[string][2]string{ "en": {"“", "”"}, "de": {"„", "“"}, Index: encoder/htmlenc/visitor.go ================================================================== --- encoder/htmlenc/visitor.go +++ encoder/htmlenc/visitor.go @@ -23,44 +23,41 @@ "zettelstore.de/z/strfun" ) // visitor writes the abstract syntax tree to an io.Writer. type visitor struct { - enc *htmlEncoder - b encoder.BufWriter - visibleSpace bool // Show space character in raw text - inVerse bool // In verse block - xhtml bool // copied from enc.xhtml - lang langStack + env *encoder.Environment + b encoder.BufWriter + visibleSpace bool // Show space character in raw text + inVerse bool // In verse block + inInteractive bool // Rendered interactive HTML code + lang langStack } func newVisitor(he *htmlEncoder, w io.Writer) *visitor { + var lang string + if he.env != nil { + lang = he.env.Lang + } return &visitor{ - enc: he, - b: encoder.NewBufWriter(w), - xhtml: he.xhtml, - lang: newLangStack(he.lang), + env: he.env, + b: encoder.NewBufWriter(w), + lang: newLangStack(lang), } } var mapMetaKey = map[string]string{ meta.KeyCopyright: "copyright", meta.KeyLicense: "license", } -func (v *visitor) acceptMeta(m *meta.Meta, withTitle bool) { - for i, pair := range m.Pairs(true) { - if v.enc.ignoreMeta[pair.Key] { +func (v *visitor) acceptMeta(m *meta.Meta) { + for _, pair := range m.Pairs(true) { + if env := v.env; env != nil && env.IgnoreMeta[pair.Key] { continue } - if i == 0 { // "title" is number 0... - if withTitle { - // TODO: title value may contain zmk elements - v.b.WriteStrings("") - } + if pair.Key == meta.KeyTitle { continue } if pair.Key == meta.KeyTags { v.writeTags(pair.Value) } else if key, ok := mapMetaKey[pair.Key]; ok { @@ -103,16 +100,17 @@ in.Accept(v) } } func (v *visitor) writeEndnotes() { - if len(v.enc.footnotes) > 0 { + footnotes := v.env.GetCleanFootnotes() + if len(footnotes) > 0 { v.b.WriteString("
    \n") - for i := 0; i < len(v.enc.footnotes); i++ { + for i := 0; i < len(footnotes); i++ { // Do not use a range loop above, because a footnote may contain // a footnote. Therefore v.enc.footnote may grow during the loop. - fn := v.enc.footnotes[i] + fn := footnotes[i] n := strconv.Itoa(i + 1) v.b.WriteStrings("
  1. ") v.acceptInlineSlice(fn.Inlines) v.b.WriteStrings( " 0 { - v.b.WriteString("\n[Header") - first := true - v.level++ - for _, p := range pairs { - if !first { - v.b.WriteByte(',') - } - v.writeNewLine() - v.b.WriteByte('[') - v.b.WriteStrings(p.Key, " \"") - v.writeEscaped(p.Value) - v.b.WriteString("\"]") - first = false - } - v.level-- - v.b.WriteByte(']') - } + pairs := m.PairsRest(true) + if len(pairs) == 0 { + return + } + v.b.WriteString("\n[Header") + v.level++ + for i, p := range pairs { + if i > 0 { + v.b.WriteByte(',') + } + v.writeNewLine() + v.b.WriteByte('[') + v.b.WriteStrings(p.Key, " \"") + v.writeEscaped(p.Value) + v.b.WriteString("\"]") + } + v.level-- + v.b.WriteByte(']') } func (v *visitor) writeMetaString(m *meta.Meta, key, native string) { if val, ok := m.Get(key); ok && len(val) > 0 { v.b.WriteStrings("\n[", native, " \"", val, "\"]") @@ -392,17 +383,14 @@ ast.RefStateExternal: "EXTERNAL", } // VisitLink writes native code for links. func (v *visitor) VisitLink(ln *ast.LinkNode) { - if adapt := v.enc.adaptLink; adapt != nil { - n := adapt(ln) - var ok bool - if ln, ok = n.(*ast.LinkNode); !ok { - n.Accept(v) - return - } + ln, n := v.env.AdaptLink(ln) + if n != nil { + n.Accept(v) + return } v.b.WriteString("Link") v.visitAttributes(ln.Attrs) v.b.WriteByte(' ') v.b.WriteString(mapRefState[ln.Ref.State]) @@ -415,17 +403,14 @@ v.b.WriteByte(']') } // VisitImage writes native code for images. func (v *visitor) VisitImage(in *ast.ImageNode) { - if adapt := v.enc.adaptImage; adapt != nil { - n := adapt(in) - var ok bool - if in, ok = n.(*ast.ImageNode); !ok { - n.Accept(v) - return - } + in, n := v.env.AdaptImage(in) + if n != nil { + n.Accept(v) + return } v.b.WriteString("Image") v.visitAttributes(in.Attrs) if in.Ref == nil { v.b.WriteStrings(" {\"", in.Syntax, "\" \"") DELETED encoder/options.go Index: encoder/options.go ================================================================== --- encoder/options.go +++ /dev/null @@ -1,76 +0,0 @@ -//----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern -// -// This file is part of zettelstore. -// -// Zettelstore is licensed under the latest version of the EUPL (European Union -// Public License). Please see file LICENSE.txt for your rights and obligations -// under this license. -//----------------------------------------------------------------------------- - -// Package encoder provides a generic interface to encode the abstract syntax -// tree into some text form. -package encoder - -import ( - "zettelstore.de/z/ast" -) - -// StringOption is an option with a string value -type StringOption struct { - Key string - Value string -} - -// Name returns the visible name of this option. -func (so *StringOption) Name() string { return so.Key } - -// BoolOption is an option with a boolean value. -type BoolOption struct { - Key string - Value bool -} - -// Name returns the visible name of this option. -func (bo *BoolOption) Name() string { return bo.Key } - -// TitleOption is an option to give the title as a AST inline slice -type TitleOption struct { - Inline ast.InlineSlice -} - -// Name returns the visible name of this option. -func (mo *TitleOption) Name() string { return "title" } - -// StringsOption is an option that have a sequence of strings as the value. -type StringsOption struct { - Key string - Value []string -} - -// Name returns the visible name of this option. -func (so *StringsOption) Name() string { return so.Key } - -// AdaptLinkOption specifies a link adapter. -type AdaptLinkOption struct { - Adapter func(*ast.LinkNode) ast.InlineNode -} - -// Name returns the visible name of this option. -func (al *AdaptLinkOption) Name() string { return "AdaptLinkOption" } - -// AdaptImageOption specifies an image adapter. -type AdaptImageOption struct { - Adapter func(*ast.ImageNode) ast.InlineNode -} - -// Name returns the visible name of this option. -func (al *AdaptImageOption) Name() string { return "AdaptImageOption" } - -// AdaptCiteOption specifies a citation adapter. -type AdaptCiteOption struct { - Adapter func(*ast.CiteNode) ast.InlineNode -} - -// Name returns the visible name of this option. -func (al *AdaptCiteOption) Name() string { return "AdaptCiteOption" } Index: encoder/rawenc/rawenc.go ================================================================== --- encoder/rawenc/rawenc.go +++ encoder/rawenc/rawenc.go @@ -19,30 +19,27 @@ "zettelstore.de/z/encoder" ) func init() { encoder.Register("raw", encoder.Info{ - Create: func() encoder.Encoder { return &rawEncoder{} }, + Create: func(*encoder.Environment) encoder.Encoder { return &rawEncoder{} }, }) } type rawEncoder struct{} -// SetOption does nothing because this encoder does not recognize any option. -func (re *rawEncoder) SetOption(option encoder.Option) {} - // WriteZettel writes the encoded zettel to the writer. func (re *rawEncoder) WriteZettel( w io.Writer, zn *ast.ZettelNode, inhMeta bool) (int, error) { b := encoder.NewBufWriter(w) if inhMeta { zn.InhMeta.Write(&b, true) } else { - zn.Zettel.Meta.Write(&b, true) + zn.Meta.Write(&b, true) } b.WriteByte('\n') - b.WriteString(zn.Zettel.Content.AsString()) + b.WriteString(zn.Content.AsString()) length, err := b.Flush() return length, err } // WriteMeta encodes meta data as HTML5. @@ -53,11 +50,11 @@ return length, err } func (re *rawEncoder) WriteContent(w io.Writer, zn *ast.ZettelNode) (int, error) { b := encoder.NewBufWriter(w) - b.WriteString(zn.Zettel.Content.AsString()) + b.WriteString(zn.Content.AsString()) length, err := b.Flush() return length, err } // WriteBlocks writes a block slice to the writer Index: encoder/textenc/textenc.go ================================================================== --- encoder/textenc/textenc.go +++ encoder/textenc/textenc.go @@ -15,42 +15,57 @@ "io" "zettelstore.de/z/ast" "zettelstore.de/z/domain/meta" "zettelstore.de/z/encoder" + "zettelstore.de/z/parser" ) func init() { encoder.Register("text", encoder.Info{ - Create: func() encoder.Encoder { return &textEncoder{} }, + Create: func(*encoder.Environment) encoder.Encoder { return &textEncoder{} }, }) } type textEncoder struct{} -// SetOption does nothing because this encoder does not recognize any option. -func (te *textEncoder) SetOption(option encoder.Option) {} - -// WriteZettel does nothing. -func (te *textEncoder) WriteZettel( - w io.Writer, zn *ast.ZettelNode, inhMeta bool) (int, error) { +// WriteZettel writes metadata and content. +func (te *textEncoder) WriteZettel(w io.Writer, zn *ast.ZettelNode, inhMeta bool) (int, error) { v := newVisitor(w) if inhMeta { te.WriteMeta(&v.b, zn.InhMeta) } else { - te.WriteMeta(&v.b, zn.Zettel.Meta) + te.WriteMeta(&v.b, zn.Meta) } v.acceptBlockSlice(zn.Ast) length, err := v.b.Flush() return length, err } -// WriteMeta encodes meta data as text. +// WriteMeta encodes metadata as text. func (te *textEncoder) WriteMeta(w io.Writer, m *meta.Meta) (int, error) { b := encoder.NewBufWriter(w) for _, pair := range m.Pairs(true) { - b.WriteString(pair.Value) + switch meta.Type(pair.Key) { + case meta.TypeBool: + if meta.BoolValue(pair.Value) { + b.WriteString("true") + } else { + b.WriteString("false") + } + case meta.TypeTagSet: + for i, tag := range meta.ListFromValue(pair.Value) { + if i > 0 { + b.WriteByte(' ') + } + b.WriteString(meta.CleanTag(tag)) + } + case meta.TypeZettelmarkup: + te.WriteInlines(w, parser.ParseMetadata(pair.Value)) + default: + b.WriteString(pair.Value) + } b.WriteByte('\n') } length, err := b.Flush() return length, err } Index: encoder/zmkenc/zmkenc.go ================================================================== --- encoder/zmkenc/zmkenc.go +++ encoder/zmkenc/zmkenc.go @@ -21,27 +21,24 @@ "zettelstore.de/z/encoder" ) func init() { encoder.Register("zmk", encoder.Info{ - Create: func() encoder.Encoder { return &zmkEncoder{} }, + Create: func(*encoder.Environment) encoder.Encoder { return &zmkEncoder{} }, }) } type zmkEncoder struct{} -// SetOption does nothing because this encoder does not recognize any option. -func (ze *zmkEncoder) SetOption(option encoder.Option) {} - // WriteZettel writes the encoded zettel to the writer. func (ze *zmkEncoder) WriteZettel( w io.Writer, zn *ast.ZettelNode, inhMeta bool) (int, error) { v := newVisitor(w, ze) if inhMeta { zn.InhMeta.WriteAsHeader(&v.b, true) } else { - zn.Zettel.Meta.WriteAsHeader(&v.b, true) + zn.Meta.WriteAsHeader(&v.b, true) } v.acceptBlockSlice(zn.Ast) length, err := v.b.Flush() return length, err } Index: go.mod ================================================================== --- go.mod +++ go.mod @@ -1,11 +1,12 @@ module zettelstore.de/z -go 1.15 +go 1.16 require ( github.com/fsnotify/fsnotify v1.4.9 github.com/pascaldekloe/jwt v1.10.0 - github.com/yuin/goldmark v1.3.2 - golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a - golang.org/x/text v0.3.0 + github.com/yuin/goldmark v1.3.3 + golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 + golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 + golang.org/x/text v0.3.6 ) Index: go.sum ================================================================== --- go.sum +++ go.sum @@ -1,16 +1,20 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/pascaldekloe/jwt v1.10.0 h1:ktcIUV4TPvh404R5dIBEnPCsSwj0sqi3/0+XafE5gJs= github.com/pascaldekloe/jwt v1.10.0/go.mod h1:TKhllgThT7TOP5rGr2zMLKEDZRAgJfBbtKyVeRsNB9A= -github.com/yuin/goldmark v1.3.2 h1:YjHC5TgyMmHpicTgEqDN0Q96Xo8K6tLXPnmNOHXCgs0= -github.com/yuin/goldmark v1.3.2/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.3.3 h1:37BdQwPx8VOSic8eDSWee6QL9mRpZRm9VJp/QugNrW0= +github.com/yuin/goldmark v1.3.3/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 h1:L2auWcuQIvxz9xSEqzESnV/QN/gNRXNApHi3fYwl2w0= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= Index: index/index.go ================================================================== --- index/index.go +++ index/index.go @@ -17,19 +17,34 @@ "time" "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" - "zettelstore.de/z/place" + "zettelstore.de/z/place/change" ) // Enricher is used to update metadata by adding new properties. type Enricher interface { // Enrich computes additional properties and updates the given metadata. // It is typically called by zettel reading methods. Enrich(ctx context.Context, m *meta.Meta) } + +// Selector is used to select zettel identifier based on selection criteria. +type Selector interface { + // Select all zettel that contains the given exact word. + // The word must be normalized through Unicode NKFD. + Select(word string) id.Set + + // Select all zettel that have a word with the given prefix. + // The prefix must be normalized through Unicode NKFD. + SelectPrefix(prefix string) id.Set + + // Select all zettel that contains the given string. + // The string must be normalized through Unicode NKFD. + SelectContains(s string) id.Set +} // NoEnrichContext will signal an enricher that nothing has to be done. // This is useful for an Indexer, but also for some place.Place calls, when // just the plain metadata is needed. func NoEnrichContext(ctx context.Context) context.Context { @@ -46,19 +61,20 @@ return ok } // Port contains all the used functions to access zettel to be indexed. type Port interface { - RegisterObserver(func(place.ChangeInfo)) + RegisterObserver(change.Func) FetchZids(context.Context) (id.Set, error) GetMeta(context.Context, id.Zid) (*meta.Meta, error) GetZettel(context.Context, id.Zid) (domain.Zettel, error) } // Indexer contains all the functions of an index. type Indexer interface { Enricher + Selector // Start the index. It will read all zettel and store index data for later retrieval. Start(Port) // Stop the index. No zettel are read any more, but the current index data @@ -87,10 +103,11 @@ // Store all relevant zettel data. There may be multiple implementations, i.e. // memory-based, file-based, based on SQLite, ... type Store interface { Enricher + Selector // UpdateReferences for a specific zettel. // Returns set of zettel identifier that must also be checked for changes. UpdateReferences(context.Context, *ZettelIndex) id.Set @@ -110,6 +127,9 @@ // Zettel is the number of zettel managed by the indexer. Zettel int // Updates count the number of metadata updates. Updates uint64 + + // Words count the different words stored in the store. + Words uint64 } ADDED index/indexer/collect.go Index: index/indexer/collect.go ================================================================== --- /dev/null +++ index/indexer/collect.go @@ -0,0 +1,126 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package indexer allows to search for metadata and content. +package indexer + +import ( + "zettelstore.de/z/ast" + "zettelstore.de/z/domain/id" + "zettelstore.de/z/index" + "zettelstore.de/z/strfun" +) + +func collectZettelIndexData(zn *ast.ZettelNode, refs id.Set, words index.WordSet) { + ixv := ixVisitor{refs: refs, words: words} + ast.NewTopDownTraverser(&ixv).VisitBlockSlice(zn.Ast) +} + +func collectInlineIndexData(ins ast.InlineSlice, refs id.Set, words index.WordSet) { + ixv := ixVisitor{refs: refs, words: words} + ast.NewTopDownTraverser(&ixv).VisitInlineSlice(ins) +} + +type ixVisitor struct { + refs id.Set + words index.WordSet +} + +// VisitVerbatim collects the verbatim text in the word set. +func (lv *ixVisitor) VisitVerbatim(vn *ast.VerbatimNode) { + for _, line := range vn.Lines { + lv.addText(line) + } +} + +// VisitRegion does nothing. +func (lv *ixVisitor) VisitRegion(rn *ast.RegionNode) {} + +// VisitHeading does nothing. +func (lv *ixVisitor) VisitHeading(hn *ast.HeadingNode) {} + +// VisitHRule does nothing. +func (lv *ixVisitor) VisitHRule(hn *ast.HRuleNode) {} + +// VisitList does nothing. +func (lv *ixVisitor) VisitNestedList(ln *ast.NestedListNode) {} + +// VisitDescriptionList does nothing. +func (lv *ixVisitor) VisitDescriptionList(dn *ast.DescriptionListNode) {} + +// VisitPara does nothing. +func (lv *ixVisitor) VisitPara(pn *ast.ParaNode) {} + +// VisitTable does nothing. +func (lv *ixVisitor) VisitTable(tn *ast.TableNode) {} + +// VisitBLOB does nothing. +func (lv *ixVisitor) VisitBLOB(bn *ast.BLOBNode) {} + +// VisitText collects the text in the word set. +func (lv *ixVisitor) VisitText(tn *ast.TextNode) { + lv.addText(tn.Text) +} + +// VisitTag collects the tag name in the word set. +func (lv *ixVisitor) VisitTag(tn *ast.TagNode) { + lv.addText(tn.Tag) +} + +// VisitSpace does nothing. +func (lv *ixVisitor) VisitSpace(sn *ast.SpaceNode) {} + +// VisitBreak does nothing. +func (lv *ixVisitor) VisitBreak(bn *ast.BreakNode) {} + +// VisitLink collects the given link as a reference. +func (lv *ixVisitor) VisitLink(ln *ast.LinkNode) { + ref := ln.Ref + if ref == nil || !ref.IsZettel() { + return + } + if zid, err := id.Parse(ref.URL.Path); err == nil { + lv.refs[zid] = true + } +} + +// VisitImage collects the image links as a reference. +func (lv *ixVisitor) VisitImage(in *ast.ImageNode) { + ref := in.Ref + if ref == nil || !ref.IsZettel() { + return + } + if zid, err := id.Parse(ref.URL.Path); err == nil { + lv.refs[zid] = true + } +} + +// VisitCite does nothing. +func (lv *ixVisitor) VisitCite(cn *ast.CiteNode) {} + +// VisitFootnote does nothing. +func (lv *ixVisitor) VisitFootnote(fn *ast.FootnoteNode) {} + +// VisitMark does nothing. +func (lv *ixVisitor) VisitMark(mn *ast.MarkNode) {} + +// VisitFormat does nothing. +func (lv *ixVisitor) VisitFormat(fn *ast.FormatNode) {} + +// VisitLiteral collects the literal words in the word set. +func (lv *ixVisitor) VisitLiteral(ln *ast.LiteralNode) { + lv.addText(ln.Text) +} + +func (lv *ixVisitor) addText(s string) { + for _, word := range strfun.NormalizeWords(s) { + lv.words[word] = lv.words[word] + 1 + } +} Index: index/indexer/indexer.go ================================================================== --- index/indexer/indexer.go +++ index/indexer/indexer.go @@ -11,22 +11,23 @@ // Package indexer allows to search for metadata and content. package indexer import ( "context" + "log" + "runtime/debug" "sync" "time" - "zettelstore.de/z/ast" - "zettelstore.de/z/collect" "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/index" "zettelstore.de/z/index/memstore" "zettelstore.de/z/parser" - "zettelstore.de/z/place" + "zettelstore.de/z/place/change" + "zettelstore.de/z/strfun" ) type indexer struct { store index.Store ar *anterooms @@ -49,17 +50,17 @@ ar: newAnterooms(10), ready: make(chan struct{}, 1), } } -func (idx *indexer) observer(ci place.ChangeInfo) { +func (idx *indexer) observer(ci change.Info) { switch ci.Reason { - case place.OnReload: + case change.OnReload: idx.ar.Reset() - case place.OnUpdate: + case change.OnUpdate: idx.ar.Enqueue(ci.Zid, arUpdate) - case place.OnDelete: + case change.OnDelete: idx.ar.Enqueue(ci.Zid, arDelete) default: return } select { @@ -98,10 +99,29 @@ return } idx.store.Enrich(ctx, m) } +// Select all zettel that contains the given exact word. +// The word must be normalized through Unicode NKFD. +func (idx *indexer) Select(word string) id.Set { + return idx.store.Select(word) +} + +// Select all zettel that have a word with the given prefix. +// The prefix must be normalized through Unicode NKFD. +func (idx *indexer) SelectPrefix(prefix string) id.Set { + return idx.store.SelectPrefix(prefix) +} + +// Select all zettel that contains the given string. +// The string must be normalized through Unicode NKFD. +func (idx *indexer) SelectContains(s string) id.Set { + return idx.store.SelectContains(s) +} + +// ReadStats populates st with indexer statistics. func (idx *indexer) ReadStats(st *index.IndexerStats) { idx.mx.RLock() st.LastReload = idx.lastReload st.IndexesSinceReload = idx.sinceReload st.DurLastIndex = idx.durLastIndex @@ -118,11 +138,13 @@ // indexer runs in the background and updates the index data structures. // This is the main service of the indexer. func (idx *indexer) indexer(p indexerPort) { // Something may panic. Ensure a running indexer. defer func() { - if err := recover(); err != nil { + if r := recover(); r != nil { + log.Println("recovered from:", r) + debug.PrintStack() go idx.indexer(p) } }() timerDuration := 15 * time.Second @@ -203,12 +225,21 @@ GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) } func (idx *indexer) updateZettel(ctx context.Context, zettel domain.Zettel, p getMetaPort) { m := zettel.Meta + if m.GetBool(meta.KeyNoIndex) { + // Zettel maybe in index + toCheck := idx.store.DeleteZettel(ctx, m.Zid) + idx.checkZettel(toCheck) + return + } + refs := id.NewSet() + words := make(index.WordSet) + collectZettelIndexData(parser.ParseZettel(zettel, ""), refs, words) zi := index.NewZettelIndex(m.Zid) - for _, pair := range m.PairsRest(false) { + for _, pair := range m.Pairs(false) { descr := meta.GetDescription(pair.Key) if descr.IsComputed() { continue } switch descr.Type { @@ -216,16 +247,26 @@ updateValue(ctx, descr.Inverse, pair.Value, p, zi) case meta.TypeIDSet: for _, val := range meta.ListFromValue(pair.Value) { updateValue(ctx, descr.Inverse, val, p, zi) } + case meta.TypeZettelmarkup: + collectInlineIndexData(parser.ParseMetadata(pair.Value), refs, words) + default: + for _, word := range strfun.NormalizeWords(pair.Value) { + words[word] = words[word] + 1 + } + } + } + for ref := range refs { + if _, err := p.GetMeta(ctx, ref); err == nil { + zi.AddBackRef(ref) + } else { + zi.AddDeadRef(ref) } } - zn := parser.ParseZettel(zettel, "") - refs := collect.References(zn) - updateReferences(ctx, refs.Links, p, zi) - updateReferences(ctx, refs.Images, p, zi) + zi.SetWords(words) toCheck := idx.store.UpdateReferences(ctx, zi) idx.checkZettel(toCheck) } func updateValue(ctx context.Context, inverse string, value string, p getMetaPort, zi *index.ZettelIndex) { @@ -242,29 +283,10 @@ return } zi.AddMetaRef(inverse, zid) } -func updateReferences(ctx context.Context, refs []*ast.Reference, p getMetaPort, zi *index.ZettelIndex) { - zrefs, _, _ := collect.DivideReferences(refs, false) - for _, ref := range zrefs { - updateReference(ctx, ref.URL.Path, p, zi) - } -} - -func updateReference(ctx context.Context, value string, p getMetaPort, zi *index.ZettelIndex) { - zid, err := id.Parse(value) - if err != nil { - return - } - if _, err := p.GetMeta(ctx, zid); err != nil { - zi.AddDeadRef(zid) - return - } - zi.AddBackRef(zid) -} - func (idx *indexer) deleteZettel(zid id.Zid) { toCheck := idx.store.DeleteZettel(context.Background(), zid) idx.checkZettel(toCheck) } Index: index/memstore/memstore.go ================================================================== --- index/memstore/memstore.go +++ index/memstore/memstore.go @@ -13,10 +13,11 @@ import ( "context" "fmt" "io" + "strings" "sync" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/index" @@ -30,42 +31,53 @@ type zettelIndex struct { dead id.Slice forward id.Slice backward id.Slice meta map[string]metaRefs + words []string } func (zi *zettelIndex) isEmpty() bool { - if len(zi.forward) > 0 || len(zi.backward) > 0 || len(zi.dead) > 0 { + if len(zi.forward) > 0 || len(zi.backward) > 0 || len(zi.dead) > 0 || len(zi.words) > 0 { return false } return zi.meta == nil || len(zi.meta) == 0 } type memStore struct { - mx sync.RWMutex - idx map[id.Zid]*zettelIndex - dead map[id.Zid]id.Slice // map dead refs where they occur + mx sync.RWMutex + idx map[id.Zid]*zettelIndex + dead map[id.Zid]id.Slice // map dead refs where they occur + words map[string]id.Slice // Stats updates uint64 } // New returns a new memory-based index store. func New() index.Store { return &memStore{ - idx: make(map[id.Zid]*zettelIndex), - dead: make(map[id.Zid]id.Slice), + idx: make(map[id.Zid]*zettelIndex), + dead: make(map[id.Zid]id.Slice), + words: make(map[string]id.Slice), } } func (ms *memStore) Enrich(ctx context.Context, m *meta.Meta) { + if ms.doEnrich(ctx, m) { + ms.mx.Lock() + ms.updates++ + ms.mx.Unlock() + } +} + +func (ms *memStore) doEnrich(ctx context.Context, m *meta.Meta) bool { ms.mx.RLock() defer ms.mx.RUnlock() zi, ok := ms.idx[m.Zid] if !ok { - return + return false } var updated bool if len(zi.dead) > 0 { m.Set(meta.KeyDead, zi.dead.String()) updated = true @@ -91,13 +103,51 @@ } if len(back) > 0 { m.Set(meta.KeyBack, back.String()) updated = true } - if updated { - ms.updates++ + return updated +} + +// Select all zettel that contains the given exact word. +// The word must be normalized through Unicode NKFD. +func (ms *memStore) Select(word string) id.Set { + ms.mx.RLock() + defer ms.mx.RUnlock() + if refs, ok := ms.words[word]; ok { + return id.NewSet(refs...) + } + return nil +} + +// Select all zettel that have a word with the given prefix. +// The prefix must be normalized through Unicode NKFD. +func (ms *memStore) SelectPrefix(prefix string) id.Set { + ms.mx.RLock() + defer ms.mx.RUnlock() + return ms.selectWithPred(prefix, strings.HasPrefix) +} + +// Select all zettel that contains the given string. +// The string must be normalized through Unicode NKFD. +func (ms *memStore) SelectContains(s string) id.Set { + ms.mx.RLock() + defer ms.mx.RUnlock() + return ms.selectWithPred(s, strings.Contains) +} + +func (ms *memStore) selectWithPred(s string, pred func(string, string) bool) id.Set { + result := id.NewSet() + for word, refs := range ms.words { + if !pred(word, s) { + continue + } + for _, ref := range refs { + result[ref] = true + } } + return result } func removeOtherMetaRefs(m *meta.Meta, back id.Slice) id.Slice { for _, p := range m.PairsRest(false) { switch meta.Type(p.Key) { @@ -134,10 +184,11 @@ } ms.updateDeadReferences(zidx, zi) ms.updateForwardBackwardReferences(zidx, zi) ms.updateMetadataReferences(zidx, zi) + ms.updateWords(zidx, zi) // Check if zi must be inserted into ms.idx if !ziExist && !zi.isEmpty() { ms.idx[zidx.Zid] = zi } @@ -201,10 +252,36 @@ bzi.meta[key] = bmr } ms.removeInverseMeta(zidx.Zid, key, remRefs) } } + +func (ms *memStore) updateWords(zidx *index.ZettelIndex, zi *zettelIndex) { + // Must only be called if ms.mx is write-locked! + words := zidx.GetWords() + newWords, removeWords := words.Diff(zi.words) + for _, word := range newWords { + if refs, ok := ms.words[word]; ok { + ms.words[word] = addRef(refs, zidx.Zid) + continue + } + ms.words[word] = id.Slice{zidx.Zid} + } + for _, word := range removeWords { + refs, ok := ms.words[word] + if !ok { + continue + } + refs2 := remRef(refs, zidx.Zid) + if len(refs2) == 0 { + delete(ms.words, word) + continue + } + ms.words[word] = refs2 + } + zi.words = words.Words() +} func (ms *memStore) getEntry(zid id.Zid) *zettelIndex { // Must only be called if ms.mx is write-locked! if zi, ok := ms.idx[zid]; ok { return zi @@ -228,10 +305,11 @@ if len(zi.meta) > 0 { for key, mrefs := range zi.meta { ms.removeInverseMeta(zid, key, mrefs.forward) } } + ms.deleteWords(zid, zi.words) delete(ms.idx, zid) return toCheck } func (ms *memStore) deleteDeadSources(zid id.Zid, zi *zettelIndex) { @@ -288,15 +366,32 @@ bzi.meta = nil } } } } + +func (ms *memStore) deleteWords(zid id.Zid, words []string) { + // Must only be called if ms.mx is write-locked! + for _, word := range words { + refs, ok := ms.words[word] + if !ok { + continue + } + refs2 := remRef(refs, zid) + if len(refs2) == 0 { + delete(ms.words, word) + continue + } + ms.words[word] = refs2 + } +} func (ms *memStore) ReadStats(st *index.StoreStats) { ms.mx.RLock() st.Zettel = len(ms.idx) st.Updates = ms.updates + st.Words = uint64(len(ms.words)) ms.mx.RUnlock() } func (ms *memStore) Write(w io.Writer) { ms.mx.RLock() ADDED index/wordset.go Index: index/wordset.go ================================================================== --- /dev/null +++ index/wordset.go @@ -0,0 +1,53 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package index allows to search for metadata and content. +package index + +// WordSet contains the set of all words, with the count of their occurrences. +type WordSet map[string]int + +// Words gives the slice of all words in the set. +func (ws WordSet) Words() []string { + if len(ws) == 0 { + return nil + } + words := make([]string, 0, len(ws)) + for w := range ws { + words = append(words, w) + } + return words +} + +// Diff calculates the word slice to be added and to be removed from oldWords +// to get the given word set. +func (ws WordSet) Diff(oldWords []string) (newWords, removeWords []string) { + if len(ws) == 0 { + return nil, oldWords + } + if len(oldWords) == 0 { + return ws.Words(), nil + } + oldSet := make(WordSet, len(oldWords)) + for _, ow := range oldWords { + if _, ok := ws[ow]; ok { + oldSet[ow] = 1 + continue + } + removeWords = append(removeWords, ow) + } + for w := range ws { + if _, ok := oldSet[w]; ok { + continue + } + newWords = append(newWords, w) + } + return newWords, removeWords +} ADDED index/wordset_test.go Index: index/wordset_test.go ================================================================== --- /dev/null +++ index/wordset_test.go @@ -0,0 +1,76 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package index allows to search for metadata and content. +package index_test + +import ( + "sort" + "testing" + + "zettelstore.de/z/index" +) + +func equalWordList(exp, got []string) bool { + if len(exp) != len(got) { + return false + } + if len(got) == 0 { + return len(exp) == 0 + } + sort.Strings(got) + for i, w := range exp { + if w != got[i] { + return false + } + } + return true +} + +func TestWordsWords(t *testing.T) { + testcases := []struct { + words index.WordSet + exp []string + }{ + {nil, nil}, + {index.WordSet{}, nil}, + {index.WordSet{"a": 1, "b": 2}, []string{"a", "b"}}, + } + for i, tc := range testcases { + got := tc.words.Words() + if !equalWordList(tc.exp, got) { + t.Errorf("%d: %v.Words() == %v, but got %v", i, tc.words, tc.exp, got) + } + } +} + +func TestWordsDiff(t *testing.T) { + testcases := []struct { + cur index.WordSet + old []string + expN, expR []string + }{ + {nil, nil, nil, nil}, + {index.WordSet{}, []string{}, nil, nil}, + {index.WordSet{"a": 1}, []string{}, []string{"a"}, nil}, + {index.WordSet{"a": 1}, []string{"b"}, []string{"a"}, []string{"b"}}, + {index.WordSet{}, []string{"b"}, nil, []string{"b"}}, + {index.WordSet{"a": 1}, []string{"a"}, nil, nil}, + } + for i, tc := range testcases { + gotN, gotR := tc.cur.Diff(tc.old) + if !equalWordList(tc.expN, gotN) { + t.Errorf("%d: %v.Diff(%v)->new %v, but got %v", i, tc.cur, tc.old, tc.expN, gotN) + } + if !equalWordList(tc.expR, gotR) { + t.Errorf("%d: %v.Diff(%v)->rem %v, but got %v", i, tc.cur, tc.old, tc.expR, gotR) + } + } +} Index: index/zettel.go ================================================================== --- index/zettel.go +++ index/zettel.go @@ -19,10 +19,11 @@ type ZettelIndex struct { Zid id.Zid // zid of the indexed zettel backrefs id.Set // set of back references metarefs map[string]id.Set // references to inverse keys deadrefs id.Set // set of dead references + words WordSet } // NewZettelIndex creates a new zettel index. func NewZettelIndex(zid id.Zid) *ZettelIndex { return &ZettelIndex{ @@ -52,26 +53,32 @@ // AddDeadRef adds a dead reference to a zettel. func (zi *ZettelIndex) AddDeadRef(zid id.Zid) { zi.deadrefs[zid] = true } +// SetWords sets the words to the given value. +func (zi *ZettelIndex) SetWords(words WordSet) { zi.words = words } + // GetDeadRefs returns all dead references as a sorted list. func (zi *ZettelIndex) GetDeadRefs() id.Slice { - return zi.deadrefs.Sort() + return zi.deadrefs.Sorted() } // GetBackRefs returns all back references as a sorted list. func (zi *ZettelIndex) GetBackRefs() id.Slice { - return zi.backrefs.Sort() + return zi.backrefs.Sorted() } // GetMetaRefs returns all meta references as a map of strings to a sorted list of references func (zi *ZettelIndex) GetMetaRefs() map[string]id.Slice { if len(zi.metarefs) == 0 { return nil } result := make(map[string]id.Slice, len(zi.metarefs)) for key, refs := range zi.metarefs { - result[key] = refs.Sort() + result[key] = refs.Sorted() } return result } + +// GetWords returns a reference to the WordSet. It must not be modified. +func (zi *ZettelIndex) GetWords() WordSet { return zi.words } Index: input/input.go ================================================================== --- input/input.go +++ input/input.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -74,10 +74,19 @@ } return r } return EOS } + +// IsEOLEOS returns true if char is either EOS or EOL. +func IsEOLEOS(ch rune) bool { + switch ch { + case EOS, '\n', '\r': + return true + } + return false +} // EatEOL transforms both "\r" and "\r\n" into "\n". func (inp *Input) EatEOL() { switch inp.Ch { case '\r': @@ -119,60 +128,69 @@ return "", false } pos := inp.Pos inp.Next() if inp.Ch == '#' { - code := 0 inp.Next() if inp.Ch == 'x' || inp.Ch == 'X' { - // Base 16 code - inp.Next() - if inp.Ch == ';' { - return "", false - } - for { - switch ch := inp.Ch; ch { - case ';': - inp.Next() - return string(rune(code)), true - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - code = 16*code + int(ch-'0') - case 'a', 'b', 'c', 'd', 'e', 'f': - code = 16*code + int(ch-'a'+10) - case 'A', 'B', 'C', 'D', 'E', 'F': - code = 16*code + int(ch-'A'+10) - default: - return "", false - } - if code > unicode.MaxRune { - return "", false - } - inp.Next() - } - } - - // Base 10 code - if inp.Ch == ';' { - return "", false - } - for { - switch ch := inp.Ch; ch { - case ';': - inp.Next() - return string(rune(code)), true - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - code = 10*code + int(ch-'0') - default: - return "", false - } - if code > unicode.MaxRune { - return "", false - } - inp.Next() - } - } - + return inp.scanEntityBase16() + } + return inp.scanEntityBase10() + } + return inp.scanEntityNamed(pos) +} + +func (inp *Input) scanEntityBase16() (string, bool) { + inp.Next() + if inp.Ch == ';' { + return "", false + } + code := 0 + for { + switch ch := inp.Ch; ch { + case ';': + inp.Next() + return string(rune(code)), true + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + code = 16*code + int(ch-'0') + case 'a', 'b', 'c', 'd', 'e', 'f': + code = 16*code + int(ch-'a'+10) + case 'A', 'B', 'C', 'D', 'E', 'F': + code = 16*code + int(ch-'A'+10) + default: + return "", false + } + if code > unicode.MaxRune { + return "", false + } + inp.Next() + } +} + +func (inp *Input) scanEntityBase10() (string, bool) { + // Base 10 code + if inp.Ch == ';' { + return "", false + } + code := 0 + for { + switch ch := inp.Ch; ch { + case ';': + inp.Next() + return string(rune(code)), true + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + code = 10*code + int(ch-'0') + default: + return "", false + } + if code > unicode.MaxRune { + return "", false + } + inp.Next() + } +} +func (inp *Input) scanEntityNamed(pos int) (string, bool) { for { switch inp.Ch { case EOS, '\n', '\r': return "", false case ';': Index: input/input_test.go ================================================================== --- input/input_test.go +++ input/input_test.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations ADDED parser/cleaner/cleaner.go Index: parser/cleaner/cleaner.go ================================================================== --- /dev/null +++ parser/cleaner/cleaner.go @@ -0,0 +1,147 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2020-2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package cleaner provides funxtions to clean up the parsed AST. +package cleaner + +import ( + "strconv" + "strings" + + "zettelstore.de/z/ast" + "zettelstore.de/z/encoder" + "zettelstore.de/z/strfun" +) + +// CleanupBlockSlice cleans the given block slice. +func CleanupBlockSlice(bs ast.BlockSlice) { + cv := &cleanupVisitor{ + textEnc: encoder.Create("text", nil), + doMark: false, + } + t := ast.NewTopDownTraverser(cv) + t.VisitBlockSlice(bs) + if cv.hasMark { + cv.doMark = true + t.VisitBlockSlice(bs) + } +} + +type cleanupVisitor struct { + textEnc encoder.Encoder + ids map[string]ast.Node + hasMark bool + doMark bool +} + +// VisitVerbatim does nothing. +func (cv *cleanupVisitor) VisitVerbatim(vn *ast.VerbatimNode) {} + +// VisitRegion does nothing. +func (cv *cleanupVisitor) VisitRegion(rn *ast.RegionNode) {} + +// VisitHeading calculates the heading slug. +func (cv *cleanupVisitor) VisitHeading(hn *ast.HeadingNode) { + if cv.doMark || hn == nil || hn.Inlines == nil { + return + } + var sb strings.Builder + _, err := cv.textEnc.WriteInlines(&sb, hn.Inlines) + if err != nil { + return + } + s := strfun.Slugify(sb.String()) + if len(s) > 0 { + hn.Slug = cv.addIdentifier(s, hn) + } +} + +// VisitHRule does nothing. +func (cv *cleanupVisitor) VisitHRule(hn *ast.HRuleNode) {} + +// VisitList does nothing. +func (cv *cleanupVisitor) VisitNestedList(ln *ast.NestedListNode) {} + +// VisitDescriptionList does nothing. +func (cv *cleanupVisitor) VisitDescriptionList(dn *ast.DescriptionListNode) {} + +// VisitPara does nothing. +func (cv *cleanupVisitor) VisitPara(pn *ast.ParaNode) {} + +// VisitTable does nothing. +func (cv *cleanupVisitor) VisitTable(tn *ast.TableNode) {} + +// VisitBLOB does nothing. +func (cv *cleanupVisitor) VisitBLOB(bn *ast.BLOBNode) {} + +// VisitText does nothing. +func (cv *cleanupVisitor) VisitText(tn *ast.TextNode) {} + +// VisitTag does nothing. +func (cv *cleanupVisitor) VisitTag(tn *ast.TagNode) {} + +// VisitSpace does nothing. +func (cv *cleanupVisitor) VisitSpace(sn *ast.SpaceNode) {} + +// VisitBreak does nothing. +func (cv *cleanupVisitor) VisitBreak(bn *ast.BreakNode) {} + +// VisitLink collects the given link as a reference. +func (cv *cleanupVisitor) VisitLink(ln *ast.LinkNode) {} + +// VisitImage collects the image links as a reference. +func (cv *cleanupVisitor) VisitImage(in *ast.ImageNode) {} + +// VisitCite does nothing. +func (cv *cleanupVisitor) VisitCite(cn *ast.CiteNode) {} + +// VisitFootnote does nothing. +func (cv *cleanupVisitor) VisitFootnote(fn *ast.FootnoteNode) {} + +// VisitMark checks for duplicate marks and changes them. +func (cv *cleanupVisitor) VisitMark(mn *ast.MarkNode) { + if mn == nil { + return + } + if !cv.doMark { + cv.hasMark = true + return + } + if mn.Text == "" { + mn.Text = cv.addIdentifier("*", mn) + return + } + mn.Text = cv.addIdentifier(mn.Text, mn) +} + +// VisitFormat does nothing. +func (cv *cleanupVisitor) VisitFormat(fn *ast.FormatNode) {} + +// VisitLiteral does nothing. +func (cv *cleanupVisitor) VisitLiteral(ln *ast.LiteralNode) {} + +func (cv *cleanupVisitor) addIdentifier(id string, node ast.Node) string { + if cv.ids == nil { + cv.ids = map[string]ast.Node{id: node} + return id + } + if n, ok := cv.ids[id]; ok && n != node { + prefix := id + "-" + for count := 1; ; count++ { + newID := prefix + strconv.Itoa(count) + if n, ok := cv.ids[newID]; !ok || n == node { + cv.ids[newID] = node + return newID + } + } + } + cv.ids[id] = node + return id +} DELETED parser/cleanup.go Index: parser/cleanup.go ================================================================== --- parser/cleanup.go +++ /dev/null @@ -1,149 +0,0 @@ -//----------------------------------------------------------------------------- -// Copyright (c) 2020-2021 Detlef Stern -// -// This file is part of zettelstore. -// -// Zettelstore is licensed under the latest version of the EUPL (European Union -// Public License). Please see file LICENSE.txt for your rights and obligations -// under this license. -//----------------------------------------------------------------------------- - -// Package parser provides a generic interface to a range of different parsers. -package parser - -import ( - "strconv" - "strings" - - "zettelstore.de/z/ast" - "zettelstore.de/z/encoder" - "zettelstore.de/z/strfun" - - // Ensure that the text encoder is available - _ "zettelstore.de/z/encoder/textenc" -) - -func cleanupBlockSlice(bs ast.BlockSlice) { - cv := &cleanupVisitor{ - textEnc: encoder.Create("text"), - doMark: false, - } - t := ast.NewTopDownTraverser(cv) - t.VisitBlockSlice(bs) - if cv.hasMark { - cv.doMark = true - t.VisitBlockSlice(bs) - } -} - -type cleanupVisitor struct { - textEnc encoder.Encoder - ids map[string]ast.Node - hasMark bool - doMark bool -} - -// VisitVerbatim does nothing. -func (cv *cleanupVisitor) VisitVerbatim(vn *ast.VerbatimNode) {} - -// VisitRegion does nothing. -func (cv *cleanupVisitor) VisitRegion(rn *ast.RegionNode) {} - -// VisitHeading calculates the heading slug. -func (cv *cleanupVisitor) VisitHeading(hn *ast.HeadingNode) { - if cv.doMark || hn == nil || hn.Inlines == nil { - return - } - var sb strings.Builder - _, err := cv.textEnc.WriteInlines(&sb, hn.Inlines) - if err != nil { - return - } - s := strfun.Slugify(sb.String()) - if len(s) > 0 { - hn.Slug = cv.addIdentifier(s, hn) - } -} - -// VisitHRule does nothing. -func (cv *cleanupVisitor) VisitHRule(hn *ast.HRuleNode) {} - -// VisitList does nothing. -func (cv *cleanupVisitor) VisitNestedList(ln *ast.NestedListNode) {} - -// VisitDescriptionList does nothing. -func (cv *cleanupVisitor) VisitDescriptionList(dn *ast.DescriptionListNode) {} - -// VisitPara does nothing. -func (cv *cleanupVisitor) VisitPara(pn *ast.ParaNode) {} - -// VisitTable does nothing. -func (cv *cleanupVisitor) VisitTable(tn *ast.TableNode) {} - -// VisitBLOB does nothing. -func (cv *cleanupVisitor) VisitBLOB(bn *ast.BLOBNode) {} - -// VisitText does nothing. -func (cv *cleanupVisitor) VisitText(tn *ast.TextNode) {} - -// VisitTag does nothing. -func (cv *cleanupVisitor) VisitTag(tn *ast.TagNode) {} - -// VisitSpace does nothing. -func (cv *cleanupVisitor) VisitSpace(sn *ast.SpaceNode) {} - -// VisitBreak does nothing. -func (cv *cleanupVisitor) VisitBreak(bn *ast.BreakNode) {} - -// VisitLink collects the given link as a reference. -func (cv *cleanupVisitor) VisitLink(ln *ast.LinkNode) {} - -// VisitImage collects the image links as a reference. -func (cv *cleanupVisitor) VisitImage(in *ast.ImageNode) {} - -// VisitCite does nothing. -func (cv *cleanupVisitor) VisitCite(cn *ast.CiteNode) {} - -// VisitFootnote does nothing. -func (cv *cleanupVisitor) VisitFootnote(fn *ast.FootnoteNode) {} - -// VisitMark checks for duplicate marks and changes them. -func (cv *cleanupVisitor) VisitMark(mn *ast.MarkNode) { - if mn == nil { - return - } - if !cv.doMark { - cv.hasMark = true - return - } - if mn.Text == "" { - mn.Text = cv.addIdentifier("*", mn) - return - } - mn.Text = cv.addIdentifier(mn.Text, mn) -} - -// VisitFormat does nothing. -func (cv *cleanupVisitor) VisitFormat(fn *ast.FormatNode) {} - -// VisitLiteral does nothing. -func (cv *cleanupVisitor) VisitLiteral(ln *ast.LiteralNode) {} - -func (cv *cleanupVisitor) addIdentifier(id string, node ast.Node) string { - if cv.ids == nil { - cv.ids = map[string]ast.Node{id: node} - return id - } - if n, ok := cv.ids[id]; ok && n != node { - prefix := id + "-" - for count := 1; ; count++ { - newID := prefix + strconv.Itoa(count) - if n, ok := cv.ids[newID]; !ok || n == node { - cv.ids[newID] = node - return newID - } - } - } - cv.ids[id] = node - return id -} Index: parser/markdown/markdown.go ================================================================== --- parser/markdown/markdown.go +++ parser/markdown/markdown.go @@ -48,11 +48,11 @@ func parseMarkdown(inp *input.Input) *mdP { source := []byte(inp.Src[inp.Pos:]) parser := gm.DefaultParser() node := parser.Parse(gmText.NewReader(source)) - textEnc := encoder.Create("text") + textEnc := encoder.Create("text", nil) return &mdP{source: source, docNode: node, textEnc: textEnc} } type mdP struct { source []byte @@ -314,16 +314,22 @@ default: panic(fmt.Sprintf("Unexpected state %v", state)) } return result } + +var ignoreAfterBS = map[byte]bool{ + '!': true, '"': true, '#': true, '$': true, '%': true, '&': true, + '\'': true, '(': true, ')': true, '*': true, '+': true, ',': true, + '-': true, '.': true, '/': true, ':': true, ';': true, '<': true, + '=': true, '>': true, '?': true, '@': true, '[': true, '\\': true, + ']': true, '^': true, '_': true, '`': true, '{': true, '|': true, + '}': true, '~': true, +} // cleanText removes backslashes from TextNodes and expands entities func cleanText(text string, cleanBS bool) string { - if text == "" { - return "" - } lastPos := 0 var sb strings.Builder for pos, ch := range text { if pos < lastPos { continue @@ -335,19 +341,14 @@ sb.WriteString(s) lastPos = pos + inp.Pos } continue } - if cleanBS && ch == '\\' && pos < len(text)-1 { - switch b := text[pos+1]; b { - case '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', - ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', - '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~': - sb.WriteString(text[lastPos:pos]) - sb.WriteByte(b) - lastPos = pos + 2 - } + if cleanBS && ch == '\\' && pos < len(text)-1 && ignoreAfterBS[text[pos+1]] { + sb.WriteString(text[lastPos:pos]) + sb.WriteByte(text[pos+1]) + lastPos = pos + 2 } } if lastPos == 0 { return text } Index: parser/parser.go ================================================================== --- parser/parser.go +++ parser/parser.go @@ -17,10 +17,11 @@ "zettelstore.de/z/ast" "zettelstore.de/z/config/runtime" "zettelstore.de/z/domain" "zettelstore.de/z/domain/meta" "zettelstore.de/z/input" + "zettelstore.de/z/parser/cleaner" ) // Info describes a single parser. // // Before ParseBlocks() or ParseInlines() is called, ensure the input stream to @@ -63,39 +64,39 @@ } // ParseBlocks parses some input and returns a slice of block nodes. func ParseBlocks(inp *input.Input, m *meta.Meta, syntax string) ast.BlockSlice { bs := Get(syntax).ParseBlocks(inp, m, syntax) - cleanupBlockSlice(bs) + cleaner.CleanupBlockSlice(bs) return bs } // ParseInlines parses some input and returns a slice of inline nodes. func ParseInlines(inp *input.Input, syntax string) ast.InlineSlice { return Get(syntax).ParseInlines(inp, syntax) } -// ParseTitle parses the title of a zettel, always as Zettelmarkup -func ParseTitle(title string) ast.InlineSlice { +// ParseMetadata parses a string as Zettelmarkup, resulting in an inline slice. +// Typically used to parse the title or other metadata of type Zettelmarkup. +func ParseMetadata(title string) ast.InlineSlice { return ParseInlines(input.NewInput(title), meta.ValueSyntaxZmk) } // ParseZettel parses the zettel based on the syntax. func ParseZettel(zettel domain.Zettel, syntax string) *ast.ZettelNode { m := zettel.Meta - inhMeta := runtime.AddDefaultValues(zettel.Meta) + inhMeta := runtime.AddDefaultValues(m) if syntax == "" { syntax, _ = inhMeta.Get(meta.KeySyntax) } - title, _ := inhMeta.Get(meta.KeyTitle) parseMeta := inhMeta if syntax == meta.ValueSyntaxNone { parseMeta = m } return &ast.ZettelNode{ - Zettel: zettel, + Meta: m, + Content: zettel.Content, Zid: m.Zid, InhMeta: inhMeta, - Title: ParseTitle(title), Ast: ParseBlocks(input.NewInput(zettel.Content.AsString()), parseMeta, syntax), } } Index: parser/zettelmark/block.go ================================================================== --- parser/zettelmark/block.go +++ parser/zettelmark/block.go @@ -52,22 +52,11 @@ switch inp.Ch { case input.EOS: return nil, false case '\n', '\r': inp.EatEOL() - for _, l := range cp.lists { - if lits := len(l.Items); lits > 0 { - l.Items[lits-1] = append(l.Items[lits-1], &nullItemNode{}) - } - } - if cp.descrl != nil { - defPos := len(cp.descrl.Descriptions) - 1 - if ldds := len(cp.descrl.Descriptions[defPos].Descriptions); ldds > 0 { - cp.descrl.Descriptions[defPos].Descriptions[ldds-1] = append( - cp.descrl.Descriptions[defPos].Descriptions[ldds-1], &nullDescriptionNode{}) - } - } + cp.cleanupListsAfterEOL() return nil, false case ':': bn, success = cp.parseColon() case '`', runeModGrave, '%': cp.clearStacked() @@ -109,10 +98,25 @@ lastPara.Inlines = append(lastPara.Inlines, pn.Inlines...) return nil, true } return pn, false } + +func (cp *zmkP) cleanupListsAfterEOL() { + for _, l := range cp.lists { + if lits := len(l.Items); lits > 0 { + l.Items[lits-1] = append(l.Items[lits-1], &nullItemNode{}) + } + } + if cp.descrl != nil { + defPos := len(cp.descrl.Descriptions) - 1 + if ldds := len(cp.descrl.Descriptions[defPos].Descriptions); ldds > 0 { + cp.descrl.Descriptions[defPos].Descriptions[ldds-1] = append( + cp.descrl.Descriptions[defPos].Descriptions[ldds-1], &nullDescriptionNode{}) + } + } +} // parseColon determines which element should be parsed. func (cp *zmkP) parseColon() (ast.BlockNode, bool) { inp := cp.inp if inp.PeekN(1) == ':' { @@ -222,25 +226,12 @@ for { posL := inp.Pos switch inp.Ch { case fch: if cp.countDelim(fch) >= cnt { - cp.clearStacked() // remove any lists defined in the region - for inp.Ch == ' ' { - inp.Next() - } - for { - switch inp.Ch { - case input.EOS, '\n', '\r': - return rn, true - } - in := cp.parseInline() - if in == nil { - return rn, true - } - rn.Inlines = append(rn.Inlines, in) - } + cp.parseRegionLastLine(rn) + return rn, true } inp.SetPos(posL) case input.EOS: return nil, false } @@ -250,10 +241,27 @@ } if !cont { lastPara, _ = bn.(*ast.ParaNode) } } +} + +func (cp *zmkP) parseRegionLastLine(rn *ast.RegionNode) { + cp.clearStacked() // remove any lists defined in the region + cp.skipSpace() + for { + switch cp.inp.Ch { + case input.EOS, '\n', '\r': + return + } + in := cp.parseInline() + if in == nil { + return + } + rn.Inlines = append(rn.Inlines, in) + } + } // parseHeading parses a head line. func (cp *zmkP) parseHeading() (hn *ast.HeadingNode, success bool) { inp := cp.inp @@ -266,17 +274,14 @@ } if inp.Ch != ' ' { return nil, false } inp.Next() - for inp.Ch == ' ' { - inp.Next() - } + cp.skipSpace() hn = &ast.HeadingNode{Level: lvl - 1} for { - switch inp.Ch { - case input.EOS, '\n', '\r': + if input.IsEOLEOS(inp.Ch) { return hn, true } in := cp.parseInline() if in == nil { return hn, true @@ -306,15 +311,33 @@ '*': ast.NestedListUnordered, '#': ast.NestedListOrdered, '>': ast.NestedListQuote, } -// parseList parses a list. +// parseNestedList parses a list. func (cp *zmkP) parseNestedList() (res ast.BlockNode, success bool) { inp := cp.inp - codes := []ast.NestedListCode{} -loopInit: + codes := cp.parseNestedListCodes() + if codes == nil { + return nil, false + } + cp.skipSpace() + if codes[len(codes)-1] != ast.NestedListQuote && input.IsEOLEOS(inp.Ch) { + return nil, false + } + + if len(codes) < len(cp.lists) { + cp.lists = cp.lists[:len(codes)] + } + ln, newLnCount := cp.buildNestedList(codes) + ln.Items = append(ln.Items, ast.ItemSlice{cp.parseLinePara()}) + return cp.cleanupParsedNestedList(newLnCount) +} + +func (cp *zmkP) parseNestedListCodes() []ast.NestedListCode { + inp := cp.inp + codes := make([]ast.NestedListCode, 0, 4) for { code, ok := mapRuneNestedList[inp.Ch] if !ok { panic(fmt.Sprintf("%q is not a region char", inp.Ch)) } @@ -321,30 +344,19 @@ codes = append(codes, code) inp.Next() switch inp.Ch { case '*', '#', '>': case ' ', input.EOS, '\n', '\r': - break loopInit + return codes default: - return nil, false - } - } - for inp.Ch == ' ' { - inp.Next() - } - if codes[len(codes)-1] != ast.NestedListQuote { - switch inp.Ch { - case input.EOS, '\n', '\r': - return nil, false + return nil } } - if len(codes) < len(cp.lists) { - cp.lists = cp.lists[:len(codes)] - } - var ln *ast.NestedListNode - newLnCount := 0 +} + +func (cp *zmkP) buildNestedList(codes []ast.NestedListCode) (ln *ast.NestedListNode, newLnCount int) { for i, code := range codes { if i < len(cp.lists) { if cp.lists[i].Code != code { ln = &ast.NestedListNode{Code: code} newLnCount++ @@ -357,11 +369,14 @@ ln = &ast.NestedListNode{Code: code} newLnCount++ cp.lists = append(cp.lists, ln) } } - ln.Items = append(ln.Items, ast.ItemSlice{cp.parseLinePara()}) + return ln, newLnCount +} + +func (cp *zmkP) cleanupParsedNestedList(newLnCount int) (res ast.BlockNode, success bool) { listDepth := len(cp.lists) for i := 0; i < newLnCount; i++ { childPos := listDepth - i - 1 parentPos := childPos - 1 if parentPos < 0 { @@ -373,10 +388,11 @@ } else { cp.lists[parentPos].Items = []ast.ItemSlice{{cp.lists[childPos]}} } } return nil, true + } // parseDefTerm parses a term of a definition list. func (cp *zmkP) parseDefTerm() (res ast.BlockNode, success bool) { inp := cp.inp @@ -383,13 +399,11 @@ inp.Next() if inp.Ch != ' ' { return nil, false } inp.Next() - for inp.Ch == ' ' { - inp.Next() - } + cp.skipSpace() descrl := cp.descrl if descrl == nil { descrl = &ast.DescriptionListNode{} cp.descrl = descrl } @@ -419,13 +433,11 @@ inp.Next() if inp.Ch != ' ' { return nil, false } inp.Next() - for inp.Ch == ' ' { - inp.Next() - } + cp.skipSpace() descrl := cp.descrl if descrl == nil || len(descrl.Descriptions) == 0 { return nil, false } defPos := len(descrl.Descriptions) - 1 @@ -564,17 +576,17 @@ // parseCell parses one single cell of a table row. func (cp *zmkP) parseCell() *ast.TableCell { inp := cp.inp var slice ast.InlineSlice for { - switch inp.Ch { - case input.EOS, '\n', '\r': + if input.IsEOLEOS(inp.Ch) { if len(slice) == 0 { return nil } - fallthrough - case '|': + return &ast.TableCell{Inlines: slice} + } + if inp.Ch == '|' { return &ast.TableCell{Inlines: slice} } slice = append(slice, cp.parseInline()) } } Index: parser/zettelmark/inline.go ================================================================== --- parser/zettelmark/inline.go +++ parser/zettelmark/inline.go @@ -123,14 +123,14 @@ } } func (cp *zmkP) parseBackslashRest() *ast.TextNode { inp := cp.inp - switch inp.Ch { - case input.EOS, '\n', '\r': + if input.IsEOLEOS(inp.Ch) { return &ast.TextNode{Text: "\\"} - case ' ': + } + if inp.Ch == ' ' { inp.Next() return &ast.TextNode{Text: "\u00a0"} } pos := inp.Pos inp.Next() @@ -177,70 +177,82 @@ } func (cp *zmkP) parseReference(closeCh rune) (ref string, ins ast.InlineSlice, ok bool) { inp := cp.inp inp.Next() - for inp.Ch == ' ' { - inp.Next() - } - hasSpace := false + cp.skipSpace() pos := inp.Pos -loop: - for { - switch inp.Ch { - case input.EOS: - return "", nil, false - case '\n', '\r', ' ': - hasSpace = true - case '|', closeCh: - break loop - } - inp.Next() + hasSpace, ok := cp.readReferenceToSep(closeCh) + if !ok { + return "", nil, false } if inp.Ch == '|' { // First part must be inline text if pos == inp.Pos { // [[| or {{| return "", nil, false } inp.SetPos(pos) - loop1: - for { - switch inp.Ch { - case input.EOS, '|': - break loop1 - } - in := cp.parseInline() - ins = append(ins, in) - } + ins = cp.parseReferenceInline() inp.Next() pos = inp.Pos } else if hasSpace { return "", nil, false } inp.SetPos(pos) - for inp.Ch == ' ' { - inp.Next() - pos = inp.Pos - } -loop2: - for { - switch inp.Ch { - case input.EOS, '\n', '\r', ' ': - return "", nil, false - case closeCh: - break loop2 - } - inp.Next() + cp.skipSpace() + pos = inp.Pos + if !cp.readReferenceToClose(closeCh) { + return "", nil, false } ref = inp.Src[pos:inp.Pos] inp.Next() if inp.Ch != closeCh { return "", nil, false } inp.Next() return ref, ins, true } + +func (cp *zmkP) readReferenceToSep(closeCh rune) (bool, bool) { + hasSpace := false + inp := cp.inp + for { + switch inp.Ch { + case input.EOS: + return false, false + case '\n', '\r', ' ': + hasSpace = true + case '|', closeCh: + return hasSpace, true + } + inp.Next() + } +} + +func (cp *zmkP) parseReferenceInline() (ins ast.InlineSlice) { + for { + switch cp.inp.Ch { + case input.EOS, '|': + return ins + } + in := cp.parseInline() + ins = append(ins, in) + } +} + +func (cp *zmkP) readReferenceToClose(closeCh rune) bool { + inp := cp.inp + for { + switch inp.Ch { + case input.EOS, '\n', '\r', ' ': + return false + case closeCh: + return true + } + inp.Next() + } +} func (cp *zmkP) parseCite() (*ast.CiteNode, bool) { inp := cp.inp inp.Next() switch inp.Ch { @@ -280,27 +292,21 @@ attrs := cp.parseAttributes(false) return &ast.FootnoteNode{Inlines: ins, Attrs: attrs}, true } func (cp *zmkP) parseLinkLikeRest() (ast.InlineSlice, bool) { - inp := cp.inp - for inp.Ch == ' ' { - inp.Next() - } + cp.skipSpace() var ins ast.InlineSlice + inp := cp.inp for inp.Ch != ']' { in := cp.parseInline() if in == nil { return nil, false } ins = append(ins, in) - if _, ok := in.(*ast.BreakNode); ok { - ch := cp.inp.Ch - switch ch { - case input.EOS, '\n', '\r': - return nil, false - } + if _, ok := in.(*ast.BreakNode); ok && input.IsEOLEOS(inp.Ch) { + return nil, false } } inp.Next() return ins, true } @@ -352,17 +358,14 @@ return nil, false } for inp.Ch == '%' { inp.Next() } - for inp.Ch == ' ' { - inp.Next() - } + cp.skipSpace() pos := inp.Pos for { - switch inp.Ch { - case input.EOS, '\n', '\r': + if input.IsEOLEOS(inp.Ch) { return &ast.LiteralNode{Code: ast.LiteralComment, Text: inp.Src[pos:inp.Pos]}, true } inp.Next() } } @@ -390,12 +393,12 @@ } inp.Next() // read 2nd formatting character if inp.Ch != fch { return nil, false } - fn := &ast.FormatNode{Code: code} inp.Next() + fn := &ast.FormatNode{Code: code} for { if inp.Ch == input.EOS { return nil, false } if inp.Ch == fch { @@ -405,15 +408,12 @@ fn.Attrs = cp.parseAttributes(false) return fn, true } fn.Inlines = append(fn.Inlines, &ast.TextNode{Text: string(fch)}) } else if in := cp.parseInline(); in != nil { - if _, ok := in.(*ast.BreakNode); ok { - switch inp.Ch { - case input.EOS, '\n', '\r': - return nil, false - } + if _, ok := in.(*ast.BreakNode); ok && input.IsEOLEOS(inp.Ch) { + return nil, false } fn.Inlines = append(fn.Inlines, in) } } } Index: parser/zettelmark/post-processor.go ================================================================== --- parser/zettelmark/post-processor.go +++ parser/zettelmark/post-processor.go @@ -81,46 +81,57 @@ } // VisitTable post-processes a table. func (pp *postProcessor) VisitTable(tn *ast.TableNode) { width := tableWidth(tn) - tn.Align = make([]ast.Alignment, 0, width) + tn.Align = make([]ast.Alignment, width) for i := 0; i < width; i++ { - tn.Align = append(tn.Align, ast.AlignDefault) + tn.Align[i] = ast.AlignDefault } if len(tn.Rows) > 0 && isHeaderRow(tn.Rows[0]) { tn.Header = tn.Rows[0] tn.Rows = tn.Rows[1:] - for pos, cell := range tn.Header { - if inlines := cell.Inlines; len(inlines) > 0 { - if textNode, ok := inlines[0].(*ast.TextNode); ok { - textNode.Text = strings.TrimPrefix(textNode.Text, "=") - } - if textNode, ok := inlines[len(inlines)-1].(*ast.TextNode); ok { - if tnl := len(textNode.Text); tnl > 0 { - if align := getAlignment(textNode.Text[tnl-1]); align != ast.AlignDefault { - tn.Align[pos] = align - textNode.Text = textNode.Text[0 : tnl-1] - } - } - } - } - } + pp.visitTableHeader(tn) } if len(tn.Header) > 0 { tn.Header = appendCells(tn.Header, width, tn.Align) for i, cell := range tn.Header { pp.processCell(cell, tn.Align[i]) } } + pp.visitTableRows(tn, width) +} + +func (pp *postProcessor) visitTableHeader(tn *ast.TableNode) { + for pos, cell := range tn.Header { + inlines := cell.Inlines + if len(inlines) == 0 { + continue + } + if textNode, ok := inlines[0].(*ast.TextNode); ok { + textNode.Text = strings.TrimPrefix(textNode.Text, "=") + } + if textNode, ok := inlines[len(inlines)-1].(*ast.TextNode); ok { + if tnl := len(textNode.Text); tnl > 0 { + if align := getAlignment(textNode.Text[tnl-1]); align != ast.AlignDefault { + tn.Align[pos] = align + textNode.Text = textNode.Text[0 : tnl-1] + } + } + } + } +} + +func (pp *postProcessor) visitTableRows(tn *ast.TableNode, width int) { for i, row := range tn.Rows { tn.Rows[i] = appendCells(row, width, tn.Align) row = tn.Rows[i] for i, cell := range row { pp.processCell(cell, tn.Align[i]) } } + } func tableWidth(tn *ast.TableNode) int { width := 0 for _, row := range tn.Rows { @@ -367,58 +378,65 @@ // // Two spaces following a break are merged into a hard break. func (pp *postProcessor) processInlineSliceCopy(ins ast.InlineSlice) int { maxPos := len(ins) for { - again := false - fromPos, toPos := 0, 0 - for fromPos < maxPos { - ins[toPos] = ins[fromPos] - fromPos++ - switch in := ins[toPos].(type) { - case *ast.TextNode: - for fromPos < maxPos { - if tn, ok := ins[fromPos].(*ast.TextNode); ok { - in.Text = in.Text + tn.Text - fromPos++ - } else { - break - } - } - case *ast.SpaceNode: - if fromPos < maxPos { - switch nn := ins[fromPos].(type) { - case *ast.BreakNode: - if len(in.Lexeme) > 1 { - nn.Hard = true - ins[toPos] = nn - fromPos++ - } - case *ast.TextNode: - if pp.inVerse { - ins[toPos] = &ast.TextNode{Text: strings.Repeat("\u00a0", len(in.Lexeme)) + nn.Text} - fromPos++ - again = true - } - } - } - case *ast.BreakNode: - if pp.inVerse { - in.Hard = true - } - } - toPos++ - } + again, toPos := pp.processInlineSliceCopyLoop(ins, maxPos) for pos := toPos; pos < maxPos; pos++ { ins[pos] = nil // Allow excess nodes to be garbage collected. } if !again { return toPos } maxPos = toPos } } + +func (pp *postProcessor) processInlineSliceCopyLoop( + ins ast.InlineSlice, maxPos int) (bool, int) { + + again := false + fromPos, toPos := 0, 0 + for fromPos < maxPos { + ins[toPos] = ins[fromPos] + fromPos++ + switch in := ins[toPos].(type) { + case *ast.TextNode: + for fromPos < maxPos { + if tn, ok := ins[fromPos].(*ast.TextNode); ok { + in.Text = in.Text + tn.Text + fromPos++ + } else { + break + } + } + case *ast.SpaceNode: + if fromPos < maxPos { + switch nn := ins[fromPos].(type) { + case *ast.BreakNode: + if len(in.Lexeme) > 1 { + nn.Hard = true + ins[toPos] = nn + fromPos++ + } + case *ast.TextNode: + if pp.inVerse { + ins[toPos] = &ast.TextNode{Text: strings.Repeat("\u00a0", len(in.Lexeme)) + nn.Text} + fromPos++ + again = true + } + } + } + case *ast.BreakNode: + if pp.inVerse { + in.Hard = true + } + } + toPos++ + } + return again, toPos +} // processInlineSliceTail removes empty text nodes, breaks and spaces at the end. func (pp *postProcessor) processInlineSliceTail(ins ast.InlineSlice, toPos int) int { for toPos > 0 { switch n := ins[toPos-1].(type) { Index: parser/zettelmark/zettelmark.go ================================================================== --- parser/zettelmark/zettelmark.go +++ parser/zettelmark/zettelmark.go @@ -76,52 +76,22 @@ key := string(inp.Src[posK:inp.Pos]) if inp.Ch != '=' { attrs[key] = "" return true } - if sameLine { - switch inp.Ch { - case input.EOS, '\n', '\r': - return false - } + if sameLine && input.IsEOLEOS(inp.Ch) { + return false } return cp.parseAttributeValue(key, attrs, sameLine) } func (cp *zmkP) parseAttributeValue( key string, attrs map[string]string, sameLine bool) bool { inp := cp.inp inp.Next() if inp.Ch == '"' { - inp.Next() - var val string - for { - switch inp.Ch { - case input.EOS: - return false - case '"': - updateAttrs(attrs, key, val) - inp.Next() - return true - case '\n', '\r': - if sameLine { - return false - } - inp.EatEOL() - val += " " - case '\\': - inp.Next() - switch inp.Ch { - case input.EOS, '\n', '\r': - return false - } - fallthrough - default: - val += string(inp.Ch) - inp.Next() - } - } + return cp.parseQuotedAttributeValue(key, attrs, sameLine) } posV := inp.Pos for { switch inp.Ch { case input.EOS: @@ -135,10 +105,43 @@ updateAttrs(attrs, key, inp.Src[posV:inp.Pos]) return true } inp.Next() } +} + +func (cp *zmkP) parseQuotedAttributeValue(key string, attrs map[string]string, sameLine bool) bool { + inp := cp.inp + inp.Next() + var val string + for { + switch inp.Ch { + case input.EOS: + return false + case '"': + updateAttrs(attrs, key, val) + inp.Next() + return true + case '\n', '\r': + if sameLine { + return false + } + inp.EatEOL() + val += " " + case '\\': + inp.Next() + switch inp.Ch { + case input.EOS, '\n', '\r': + return false + } + fallthrough + default: + val += string(inp.Ch) + inp.Next() + } + } + } func updateAttrs(attrs map[string]string, key, val string) { if prevVal := attrs[key]; len(prevVal) > 0 { attrs[key] = prevVal + " " + val @@ -163,11 +166,11 @@ if pos < inp.Pos { return &ast.Attributes{Attrs: map[string]string{"": inp.Src[pos:inp.Pos]}} } // No immediate name: skip spaces - cp.skipSpace(!sameLine) + cp.skipSpace() } pos := inp.Pos attrs, success := cp.doParseAttributes(sameLine) if sameLine || success { @@ -182,72 +185,83 @@ if inp.Ch != '{' { return nil, false } inp.Next() attrs := map[string]string{} -loop: + if !cp.parseAttributeValues(sameLine, attrs) { + return nil, false + } + inp.Next() + return &ast.Attributes{Attrs: attrs}, true +} + +func (cp *zmkP) parseAttributeValues(sameLine bool, attrs map[string]string) bool { + inp := cp.inp for { - cp.skipSpace(!sameLine) + cp.skipSpaceLine(sameLine) switch inp.Ch { case input.EOS: - return nil, false + return false case '}': - break loop + return true case '.': inp.Next() posC := inp.Pos for isNameRune(inp.Ch) { inp.Next() } if posC == inp.Pos { - return nil, false + return false } updateAttrs(attrs, "class", inp.Src[posC:inp.Pos]) case '=': delete(attrs, "") if !cp.parseAttributeValue("", attrs, sameLine) { - return nil, false + return false } default: if !cp.parseNormalAttribute(attrs, sameLine) { - return nil, false + return false } } + switch inp.Ch { case '}': - break loop + return true case '\n', '\r': if sameLine { - return nil, false + return false } case ' ', ',': inp.Next() default: - return nil, false - } - } - inp.Next() - return &ast.Attributes{Attrs: attrs}, true -} - -func (cp *zmkP) skipSpace(eolIsSpace bool) { - inp := cp.inp - if eolIsSpace { - for { - switch inp.Ch { - case ' ': - inp.Next() - case '\n', '\r': - inp.EatEOL() - default: - return - } - } - } - for inp.Ch == ' ' { + return false + } + } +} + +func (cp *zmkP) skipSpaceLine(sameLine bool) { + if sameLine { + cp.skipSpace() + return + } + for inp := cp.inp; ; { + switch inp.Ch { + case ' ': + inp.Next() + case '\n', '\r': + inp.EatEOL() + default: + return + } + } +} + +func (cp *zmkP) skipSpace() { + for inp := cp.inp; inp.Ch == ' '; { inp.Next() } } func isNameRune(ch rune) bool { return unicode.IsLetter(ch) || unicode.IsDigit(ch) || ch == '-' || ch == '_' } Index: parser/zettelmark/zettelmark_test.go ================================================================== --- parser/zettelmark/zettelmark_test.go +++ parser/zettelmark/zettelmark_test.go @@ -19,10 +19,14 @@ "zettelstore.de/z/ast" "zettelstore.de/z/domain/meta" "zettelstore.de/z/input" "zettelstore.de/z/parser" + + // Ensure that the text encoder is available. + // Needed by parser/cleanup.go + _ "zettelstore.de/z/encoder/textenc" ) type TestCase struct{ source, want string } type TestCases []TestCase ADDED place/change/change.go Index: place/change/change.go ================================================================== --- /dev/null +++ place/change/change.go @@ -0,0 +1,43 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package change provides definition for place changes. +package change + +import ( + "zettelstore.de/z/domain/id" +) + +// Reason gives an indication, why the ObserverFunc was called. +type Reason int + +// Values for Reason +const ( + _ Reason = iota + OnReload // Place was reloaded + OnUpdate // A zettel was created or changed + OnDelete // A zettel was removed +) + +// Info contains all the data about a changed zettel. +type Info struct { + Reason Reason + Zid id.Zid +} + +// Func is a function to be called when a change is detected. +type Func func(Info) + +// Subject is a place that notifies observers about changes. +type Subject interface { + // RegisterObserver registers an observer that will be notified + // if one or all zettel are found to be changed. + RegisterObserver(Func) +} ADDED place/constplace/base.css Index: place/constplace/base.css ================================================================== --- /dev/null +++ place/constplace/base.css @@ -0,0 +1,279 @@ +*,*::before,*::after { + box-sizing: border-box; + } + html { + font-size: 1rem; + font-family: serif; + scroll-behavior: smooth; + height: 100%; + } + body { + margin: 0; + min-height: 100vh; + text-rendering: optimizeSpeed; + line-height: 1.4; + overflow-x: hidden; + background-color: #f8f8f8 ; + height: 100%; + } + nav.zs-menu { + background-color: hsl(210, 28%, 90%); + overflow: auto; + white-space: nowrap; + font-family: sans-serif; + padding-left: .5rem; + } + nav.zs-menu > a { + float:left; + display: block; + text-align: center; + padding:.41rem .5rem; + text-decoration: none; + color:black; + } + nav.zs-menu > a:hover, .zs-dropdown:hover button { + background-color: hsl(210, 28%, 80%); + } + nav.zs-menu form { + float: right; + } + nav.zs-menu form input[type=text] { + padding: .12rem; + border: none; + margin-top: .25rem; + margin-right: .5rem; + } + .zs-dropdown { + float: left; + overflow: hidden; + } + .zs-dropdown > button { + font-size: 16px; + border: none; + outline: none; + color: black; + padding:.41rem .5rem; + background-color: inherit; + font-family: inherit; + margin: 0; + } + .zs-dropdown-content { + display: none; + position: absolute; + background-color: #f9f9f9; + min-width: 160px; + box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2); + z-index: 1; + } + .zs-dropdown-content > a { + float: none; + color: black; + padding:.41rem .5rem; + text-decoration: none; + display: block; + text-align: left; + } + .zs-dropdown-content > a:hover { + background-color: hsl(210, 28%, 75%); + } + .zs-dropdown:hover > .zs-dropdown-content { + display: block; + } + main { + padding: 0 1rem; + } + article > * + * { + margin-top: .5rem; + } + article header { + padding: 0; + margin: 0; + } + h1,h2,h3,h4,h5,h6 { font-family:sans-serif; font-weight:normal } + h1 { font-size:1.5rem; margin:.65rem 0 } + h2 { font-size:1.25rem; margin:.70rem 0 } + h3 { font-size:1.15rem; margin:.75rem 0 } + h4 { font-size:1.05rem; margin:.8rem 0; font-weight: bold } + h5 { font-size:1.05rem; margin:.8rem 0 } + h6 { font-size:1.05rem; margin:.8rem 0; font-weight: lighter } + p { + margin: .5rem 0 0 0; + } + ol,ul { + padding-left: 1.1rem; + } + li,figure,figcaption,dl { + margin: 0; + } + dt { + margin: .5rem 0 0 0; + } + dt+dd { + margin-top: 0; + } + dd { + margin: .5rem 0 0 2rem; + } + dd > p:first-child { + margin: 0 0 0 0; + } + blockquote { + border-left: 0.5rem solid lightgray; + padding-left: 1rem; + margin-left: 1rem; + margin-right: 2rem; + font-style: italic; + } + blockquote p { + margin-bottom: .5rem; + } + blockquote cite { + font-style: normal; + } + table { + border-collapse: collapse; + border-spacing: 0; + max-width: 100%; + } + th,td { + text-align: left; + padding: .25rem .5rem; + } + td { border-bottom: 1px solid hsl(0, 0%, 85%); } + thead th { border-bottom: 2px solid hsl(0, 0%, 70%); } + tfoot th { border-top: 2px solid hsl(0, 0%, 70%); } + main form { + padding: 0 .5em; + margin: .5em 0 0 0; + } + main form:after { + content: "."; + display: block; + height: 0; + clear: both; + visibility: hidden; + } + main form div { + margin: .5em 0 0 0 + } + input { + font-family: monospace; + } + input[type="submit"],button,select { + font: inherit; + } + label { font-family: sans-serif; font-size:.9rem } + label::after { content:":" } + textarea { + font-family: monospace; + resize: vertical; + width: 100%; + } + .zs-input { + padding: .5em; + display:block; + border:none; + border-bottom:1px solid #ccc; + width:100%; + } + .zs-button { + float:right; + margin: .5em 0 .5em 1em; + } + a:not([class]) { + text-decoration-skip-ink: auto; + } + .zs-broken { + text-decoration: line-through; + } + img { + max-width: 100%; + } + .zs-endnotes { + padding-top: .5rem; + border-top: 1px solid; + } + code,pre,kbd { + font-family: monospace; + font-size: 85%; + } + code { + padding: .1rem .2rem; + background: #f0f0f0; + border: 1px solid #ccc; + border-radius: .25rem; + } + pre { + padding: .5rem .7rem; + max-width: 100%; + overflow: auto; + border: 1px solid #ccc; + border-radius: .5rem; + background: #f0f0f0; + } + pre code { + font-size: 95%; + position: relative; + padding: 0; + border: none; + } + div.zs-indication { + padding: .5rem .7rem; + max-width: 100%; + border-radius: .5rem; + border: 1px solid black; + } + div.zs-indication p:first-child { + margin-top: 0; + } + span.zs-indication { + border: 1px solid black; + border-radius: .25rem; + padding: .1rem .2rem; + font-size: 95%; + } + .zs-example { border-style: dotted !important } + .zs-error { + background-color: lightpink; + border-style: none !important; + font-weight: bold; + } + kbd { + background: hsl(210, 5%, 100%); + border: 1px solid hsl(210, 5%, 70%); + border-radius: .25rem; + padding: .1rem .2rem; + font-size: 75%; + } + .zs-meta { + font-size:.75rem; + color:#444; + margin-bottom:1rem; + } + .zs-meta a { + color:#444; + } + h1+.zs-meta { + margin-top:-1rem; + } + details > summary { + width: 100%; + background-color: #eee; + font-family:sans-serif; + } + details > ul { + margin-top:0; + padding-left:2rem; + background-color: #eee; + } + footer { + padding: 0 1rem; + } + @media (prefers-reduced-motion: reduce) { + * { + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + transition-duration: 0.01ms !important; + scroll-behavior: auto !important; + } + } ADDED place/constplace/base.mustache Index: place/constplace/base.mustache ================================================================== --- /dev/null +++ place/constplace/base.mustache @@ -0,0 +1,64 @@ + + + + + + + +{{{MetaHeader}}} + +{{Title}} + + + +
    +{{{Content}}} +
    +{{#FooterHTML}} +
    +{{{FooterHTML}}} +
    +{{/FooterHTML}} + + DELETED place/constplace/constdata.go Index: place/constplace/constdata.go ================================================================== --- place/constplace/constdata.go +++ /dev/null @@ -1,773 +0,0 @@ -//----------------------------------------------------------------------------- -// Copyright (c) 2020-2021 Detlef Stern -// -// This file is part of zettelstore. -// -// Zettelstore is licensed under the latest version of the EUPL (European Union -// Public License). Please see file LICENSE.txt for your rights and obligations -// under this license. -//----------------------------------------------------------------------------- - -// Package constplace stores zettel inside the executable. -package constplace - -import ( - "zettelstore.de/z/domain/id" - "zettelstore.de/z/domain/meta" -) - -const ( - syntaxTemplate = "mustache" -) - -var constZettelMap = map[id.Zid]constZettel{ - id.ConfigurationZid: { - constHeader{ - meta.KeyTitle: "Zettelstore Runtime Configuration", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityOwner, - meta.KeySyntax: meta.ValueSyntaxNone, - }, - "", - }, - - id.BaseTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore Base HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - ` - - - - - - -{{{MetaHeader}}} - -{{Title}} - - - -
    -{{{Content}}} -
    -{{#FooterHTML}} -
    -{{{FooterHTML}}} -
    -{{/FooterHTML}} - -`}, - - id.LoginTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore Login Form HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - `
    -
    -

    {{Title}}

    -
    -{{#Retry}} -
    Wrong user name / password. Try again.
    -{{/Retry}} -
    -
    - - -
    -
    - - -
    - -
    -
    `}, - - id.ListTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore List Meta HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - ``}, - - id.DetailTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore Detail HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - `
    -
    -

    {{{HTMLTitle}}}

    -
    -{{#CanWrite}}Edit ·{{/CanWrite}} -{{Zid}} · -Info · -({{RoleText}}) -{{#HasTags}}· {{#Tags}} {{Text}}{{/Tags}}{{/HasTags}} -{{#CanCopy}}· Copy{{/CanCopy}} -{{#CanFolge}}· Folge{{/CanFolge}} -{{#FolgeRefs}}
    Folge: {{{FolgeRefs}}}{{/FolgeRefs}} -{{#PrecursorRefs}}
    Precursor: {{{PrecursorRefs}}}{{/PrecursorRefs}} -{{#HasExtURL}}
    URL: {{ExtURL}}{{/HasExtURL}} -
    -
    -{{{Content}}} -{{#HasBackLinks}} -
    -Links to this zettel -
      -{{#BackLinks}} -
    • {{Text}}
      • -{{/BackLinks}} -
    -
    -{{/HasBackLinks}} -
    `}, - - id.InfoTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore Info HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - `
    -
    -

    Information for Zettel {{Zid}}

    -Web -· Context -{{#CanWrite}} · Edit{{/CanWrite}} -{{#CanFolge}} · Folge{{/CanFolge}} -{{#CanCopy}} · Copy{{/CanCopy}} -{{#CanRename}}· Rename{{/CanRename}} -{{#CanDelete}}· Delete{{/CanDelete}} -
    -

    Interpreted Metadata

    -{{#MetaData}}{{/MetaData}}
    {{Key}}{{{Value}}}
    -{{#HasLinks}} -

    References

    -{{#HasLocLinks}} -

    Local

    -
      -{{#LocLinks}} -
    • {{.}}
      • -{{/LocLinks}} -
    -{{/HasLocLinks}} -{{#HasExtLinks}} -

    External

    -
      -{{#ExtLinks}} -
    • {{.}}
      • -{{/ExtLinks}} -
    -{{/HasExtLinks}} -{{/HasLinks}} -

    Parts and format

    - -{{#Matrix}} - -{{#Elements}}{{#HasURL}}{{/HasURL}}{{^HasURL}}{{/HasURL}} -{{/Elements}} - -{{/Matrix}} -
    {{Text}}{{Text}}
    -
    `}, - - id.ContextTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore Context HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - ``}, - - id.FormTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore Form HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - `
    -
    -

    {{Heading}}

    -
    -
    -
    - - -
    -
    -
    - - -
    - - -
    -
    - - -
    -
    - - -
    -
    -{{#IsTextContent}} - - -{{/IsTextContent}} -
    - -
    -
    `, - }, - - id.RenameTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore Rename Form HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - `
    -
    -

    Rename Zettel {{.Zid}}

    -
    -

    Do you really want to rename this zettel?

    -
    -
    - - -
    - - -
    -
    -{{#MetaPairs}} -
    {{Key}}:
    {{Value}}
    -{{/MetaPairs}} -
    -
    `, - }, - - id.DeleteTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore Delete HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - `
    -
    -

    Delete Zettel {{Zid}}

    -
    -

    Do you really want to delete this zettel?

    -
    -{{#MetaPairs}} -
    {{Key}}:
    {{Value}}
    -{{/MetaPairs}} -
    -
    - -
    -
    -{{end}}`, - }, - - id.RolesTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore List Roles HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - ``}, - - id.TagsTemplateZid: { - constHeader{ - meta.KeyTitle: "Zettelstore List Tags HTML Template", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityExpert, - meta.KeySyntax: syntaxTemplate, - }, - ``}, - - id.BaseCSSZid: { - constHeader{ - meta.KeyTitle: "Zettelstore Base CSS", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeyVisibility: meta.ValueVisibilityPublic, - meta.KeySyntax: "css", - }, - `/* Default CSS */ -*,*::before,*::after { - box-sizing: border-box; -} -html { - font-size: 1rem; - font-family: serif; - scroll-behavior: smooth; - height: 100%; -} -body { - margin: 0; - min-height: 100vh; - text-rendering: optimizeSpeed; - line-height: 1.4; - overflow-x: hidden; - background-color: #f8f8f8 ; - height: 100%; -} -nav.zs-menu { - background-color: hsl(210, 28%, 90%); - overflow: auto; - white-space: nowrap; - font-family: sans-serif; - padding-left: .5rem; -} -nav.zs-menu > a { - float:left; - display: inline-block; - text-align: center; - padding:.41rem .5rem; - text-decoration: none; - color:black; -} -nav.zs-menu > a:hover, .zs-dropdown:hover button { - background-color: hsl(210, 28%, 80%); -} -nav.zs-menu form { - float: right; -} -nav.zs-menu form input[type=text] { - padding: .12rem; - border: none; - margin-top: .25rem; - margin-right: .5rem; -} -.zs-dropdown { - float: left; - overflow: hidden; -} -.zs-dropdown > button { - font-size: 16px; - border: none; - outline: none; - color: black; - padding:.41rem .5rem; - background-color: inherit; - font-family: inherit; - margin: 0; -} -.zs-dropdown-content { - display: none; - position: absolute; - background-color: #f9f9f9; - min-width: 160px; - box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2); - z-index: 1; -} -.zs-dropdown-content > a { - float: none; - color: black; - padding:.41rem .5rem; - text-decoration: none; - display: block; - text-align: left; -} -.zs-dropdown-content > a:hover { - background-color: hsl(210, 28%, 75%); -} -.zs-dropdown:hover > .zs-dropdown-content { - display: block; -} -main { - padding: 0 1rem; -} -article > * + * { - margin-top: .5rem; -} -article header { - padding: 0; - margin: 0; -} -h1,h2,h3,h4,h5,h6 { font-family:sans-serif; font-weight:normal } -h1 { font-size:1.5rem; margin:.65rem 0 } -h2 { font-size:1.25rem; margin:.70rem 0 } -h3 { font-size:1.15rem; margin:.75rem 0 } -h4 { font-size:1.05rem; margin;.8rem 0; font-weight: bold } -h5 { font-size:1.05rem; margin;.8rem 0 } -h6 { font-size:1.05rem; margin;.8rem 0; font-weight: lighter } -p { - margin: .5rem 0 0 0; -} -ol,ul { - padding-left: 1.1rem; -} -li,figure,figcaption,dl { - margin: 0; -} -dt { - margin: .5rem 0 0 0; -} -dt+dd { - margin-top: 0; -} -dd { - margin: .5rem 0 0 2rem; -} -dd > p:first-child { - margin: 0 0 0 0; -} -blockquote { - border-left: 0.5rem solid lightgray; - padding-left: 1rem; - margin-left: 1rem; - margin-right: 2rem; - font-style: italic; -} -blockquote p { - margin-bottom: .5rem; -} -blockquote cite { - font-style: normal; -} -table { - border-collapse: collapse; - border-spacing: 0; - max-width: 100%; -} -th,td { - text-align: left; - padding: .25rem .5rem; -} -td { border-bottom: 1px solid hsl(0, 0%, 85%); } -thead th { border-bottom: 2px solid hsl(0, 0%, 70%); } -tfoot th { border-top: 2px solid hsl(0, 0%, 70%); } -main form { - padding: 0 .5em; - margin: .5em 0 0 0; -} -main form:after { - content: "."; - display: block; - height: 0; - clear: both; - visibility: hidden; -} -main form div { - margin: .5em 0 0 0 -} -input { - font-family: monospace; -} -input[type="submit"],button,select { - font: inherit; -} -label { font-family: sans-serif; font-size:.9rem } -label::after { content:":" } -textarea { - font-family: monospace; - resize: vertical; - width: 100%; -} -.zs-input { - padding: .5em; - display:block; - border:none; - border-bottom:1px solid #ccc; - width:100%; -} -.zs-button { - float:right; - margin: .5em 0 .5em 1em; -} -a:not([class]) { - text-decoration-skip-ink: auto; -} -.zs-broken { - text-decoration: line-through; -} -img { - max-width: 100%; -} -.zs-endnotes { - padding-top: .5rem; - border-top: 1px solid; -} -code,pre,kbd { - font-family: monospace; - font-size: 85%; -} -code { - padding: .1rem .2rem; - background: #f0f0f0; - border: 1px solid #ccc; - border-radius: .25rem; -} -pre { - padding: .5rem .7rem; - max-width: 100%; - overflow: auto; - border: 1px solid #ccc; - border-radius: .5rem; - background: #f0f0f0; -} -pre code { - font-size: 95%; - position: relative; - padding: 0; - border: none; -} -div.zs-indication { - padding: .5rem .7rem; - max-width: 100%; - border-radius: .5rem; - border: 1px solid black; -} -div.zs-indication p:first-child { - margin-top: 0; -} -span.zs-indication { - border: 1px solid black; - border-radius: .25rem; - padding: .1rem .2rem; - font-size: 95%; -} -.zs-example { border-style: dotted !important } -.zs-error { - background-color: lightpink; - border-style: none !important; - font-weight: bold; -} -kbd { - background: hsl(210, 5%, 100%); - border: 1px solid hsl(210, 5%, 70%); - border-radius: .25rem; - padding: .1rem .2rem; - font-size: 75%; -} -.zs-meta { - font-size:.75rem; - color:#444; - margin-bottom:1rem; -} -.zs-meta a { - color:#444; -} -h1+.zs-meta { - margin-top:-1rem; -} -details > summary { - width: 100%; - background-color: #eee; - font-family:sans-serif; -} -details > ul { - margin-top:0; - padding-left:2rem; - background-color: #eee; -} -footer { - padding: 0 1rem; -} -@media (prefers-reduced-motion: reduce) { - * { - animation-duration: 0.01ms !important; - animation-iteration-count: 1 !important; - transition-duration: 0.01ms !important; - scroll-behavior: auto !important; - } -}`}, - - id.TOCNewTemplateZid: { - constHeader{ - meta.KeyTitle: "New Menu", - meta.KeyRole: meta.ValueRoleConfiguration, - meta.KeySyntax: meta.ValueSyntaxZmk, - }, - `This zettel lists all zettel that should act as a template for new zettel. -These zettel will be included in the ""New"" menu of the WebUI. -* [[New Zettel|00000000090001]] -* [[New User|00000000090002]]`}, - - id.TemplateNewZettelZid: { - constHeader{ - meta.KeyTitle: "New Zettel", - meta.KeyRole: meta.ValueRoleZettel, - meta.KeySyntax: meta.ValueSyntaxZmk, - }, - ""}, - - id.TemplateNewUserZid: { - constHeader{ - meta.KeyTitle: "New User", - meta.KeyRole: meta.ValueRoleUser, - meta.KeyCredential: "", - meta.KeyUserID: "", - meta.KeyUserRole: meta.ValueUserRoleReader, - meta.KeySyntax: meta.ValueSyntaxNone, - }, - ""}, - - id.DefaultHomeZid: { - constHeader{ - meta.KeyTitle: "Home", - meta.KeyRole: meta.ValueRoleZettel, - meta.KeySyntax: meta.ValueSyntaxZmk, - }, - `=== Thank you for using Zettelstore! - -You will find the lastest information about Zettelstore at [[https://zettelstore.de]]. -Check that website regulary for [[upgrades|https://zettelstore.de/home/doc/trunk/www/download.wiki]] to the latest version. -You should consult the [[change log|https://zettelstore.de/home/doc/trunk/www/changes.wiki]] before upgrading. -Sometimes, you have to edit some of your Zettelstore-related zettel before upgrading. -Since Zettelstore is currently in a development state, every upgrade might fix some of your problems. -To check for versions, there is a zettel with the [[current version|00000000000001]] of your Zettelstore. - -If you have problems concerning Zettelstore, -do not hesitate to get in [[contact with the main developer|mailto:ds@zettelstore.de]]. - -=== Reporting errors -If you have encountered an error, please include the content of the following zettel in your mail: -* [[Zettelstore Version|00000000000001]] -* [[Zettelstore Operating System|00000000000003]] -* [[Zettelstore Startup Configuration|00000000000096]] -* [[Zettelstore Startup Values|00000000000098]] -* [[Zettelstore Runtime Configuration|00000000000100]] - -Additionally, you have to describe, what you have done before that error occurs -and what you have expected instead. -Please do not forget to include the error message, if there is one. - -Some of above Zettelstore zettel can only be retrieved if you enabled ""expert mode"". -Otherwise, only some zettel are linked. -To enable expert mode, edit the zettel [[Zettelstore Runtime Configuration|00000000000100]]: -please set the metadata value of the key ''expert-mode'' to true. -To do you, enter the string ''expert-mode:true'' inside the edit view of the metadata. - -=== Information about this zettel -This zettel is your home zettel. -It is part of the Zettelstore software itself. -Every time you click on the [[Home|//]] link in the menu bar, you will be redirected to this zettel. - -You can change the content of this zettel by clicking on ""Edit"" above. -This allows you to customize your home zettel. - -Alternatively, you can designate another zettel as your home zettel. -Edit the [[Zettelstore Runtime Configuration|00000000000100]] by adding the metadata key ''home-zettel''. -Its value is the identifier of the zettel that should act as the new home zettel. -You will find the identifier of each zettel between their ""Edit"" and the ""Info"" link above. -The identifier of this zettel is ''00010000000000''. -If you provide a wrong identifier, this zettel will be shown as the home zettel. -Take a look inside the manual for further details. -`}, -} Index: place/constplace/constplace.go ================================================================== --- place/constplace/constplace.go +++ place/constplace/constplace.go @@ -11,24 +11,26 @@ // Package constplace places zettel inside the executable. package constplace import ( "context" + _ "embed" // Allow to embed file content "net/url" "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/index" "zettelstore.de/z/place" "zettelstore.de/z/place/manager" + "zettelstore.de/z/search" ) func init() { manager.Register( " const", - func(u *url.URL, cdata *manager.ConnectData) (place.Place, error) { + func(u *url.URL, cdata *manager.ConnectData) (place.ManagedPlace, error) { return &constPlace{zettel: constZettelMap, filter: cdata.Filter}, nil }) } type constHeader map[string]string @@ -55,17 +57,15 @@ return "const:" } func (cp *constPlace) CanCreateZettel(ctx context.Context) bool { return false } -func (cp *constPlace) CreateZettel( - ctx context.Context, zettel domain.Zettel) (id.Zid, error) { +func (cp *constPlace) CreateZettel(ctx context.Context, zettel domain.Zettel) (id.Zid, error) { return id.Invalid, place.ErrReadOnly } -func (cp *constPlace) GetZettel( - ctx context.Context, zid id.Zid) (domain.Zettel, error) { +func (cp *constPlace) GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error) { if z, ok := cp.zettel[zid]; ok { return domain.Zettel{Meta: makeMeta(zid, z.header), Content: z.content}, nil } return domain.Zettel{}, place.ErrNotFound } @@ -83,21 +83,19 @@ result[zid] = true } return result, nil } -func (cp *constPlace) SelectMeta( - ctx context.Context, f *place.Filter, s *place.Sorter) (res []*meta.Meta, err error) { - hasMatch := place.CreateFilterFunc(f) +func (cp *constPlace) SelectMeta(ctx context.Context, match search.MetaMatchFunc) (res []*meta.Meta, err error) { for zid, zettel := range cp.zettel { m := makeMeta(zid, zettel.header) cp.filter.Enrich(ctx, m) - if hasMatch(m) { + if match(m) { res = append(res, m) } } - return place.ApplySorter(res, s), nil + return res, nil } func (cp *constPlace) CanUpdateZettel(ctx context.Context, zettel domain.Zettel) bool { return false } @@ -128,5 +126,212 @@ func (cp *constPlace) ReadStats(st *place.Stats) { st.ReadOnly = true st.Zettel = len(cp.zettel) } + +const syntaxTemplate = "mustache" + +var constZettelMap = map[id.Zid]constZettel{ + id.ConfigurationZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Runtime Configuration", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityOwner, + meta.KeySyntax: meta.ValueSyntaxNone, + meta.KeyNoIndex: meta.ValueTrue, + }, + ""}, + id.BaseTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Base HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentBaseMustache)}, + id.LoginTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Login Form HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentLoginMustache)}, + id.ZettelTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Zettel HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentZettelMustache)}, + id.InfoTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Info HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentInfoMustache)}, + id.ContextTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Context HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentContextMustache)}, + id.FormTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Form HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentFormMustache)}, + id.RenameTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Rename Form HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentRenameMustache)}, + id.DeleteTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Delete HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentDeleteMustache)}, + id.ListTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore List Zettel HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentListZettelMustache)}, + id.RolesTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore List Roles HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentListRolesMustache)}, + id.TagsTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore List Tags HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentListTagsMustache)}, + id.ErrorTemplateZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Error HTML Template", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityExpert, + meta.KeySyntax: syntaxTemplate, + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentErrorMustache)}, + id.BaseCSSZid: { + constHeader{ + meta.KeyTitle: "Zettelstore Base CSS", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeyVisibility: meta.ValueVisibilityPublic, + meta.KeySyntax: "css", + meta.KeyNoIndex: meta.ValueTrue, + }, + domain.NewContent(contentBaseCSS)}, + id.TOCNewTemplateZid: { + constHeader{ + meta.KeyTitle: "New Menu", + meta.KeyRole: meta.ValueRoleConfiguration, + meta.KeySyntax: meta.ValueSyntaxZmk, + }, + domain.NewContent(contentNewTOCZettel)}, + id.TemplateNewZettelZid: { + constHeader{ + meta.KeyTitle: "New Zettel", + meta.KeyRole: meta.ValueRoleZettel, + meta.KeySyntax: meta.ValueSyntaxZmk, + }, + ""}, + id.TemplateNewUserZid: { + constHeader{ + meta.KeyTitle: "New User", + meta.KeyRole: meta.ValueRoleUser, + meta.KeyCredential: "", + meta.KeyUserID: "", + meta.KeyUserRole: meta.ValueUserRoleReader, + meta.KeySyntax: meta.ValueSyntaxNone, + }, + ""}, + id.DefaultHomeZid: { + constHeader{ + meta.KeyTitle: "Home", + meta.KeyRole: meta.ValueRoleZettel, + meta.KeySyntax: meta.ValueSyntaxZmk, + }, + domain.NewContent(contentHomeZettel)}, +} + +//go:embed base.mustache +var contentBaseMustache string + +//go:embed login.mustache +var contentLoginMustache string + +//go:embed zettel.mustache +var contentZettelMustache string + +//go:embed info.mustache +var contentInfoMustache string + +//go:embed context.mustache +var contentContextMustache string + +//go:embed form.mustache +var contentFormMustache string + +//go:embed rename.mustache +var contentRenameMustache string + +//go:embed delete.mustache +var contentDeleteMustache string + +//go:embed listzettel.mustache +var contentListZettelMustache string + +//go:embed listroles.mustache +var contentListRolesMustache string + +//go:embed listtags.mustache +var contentListTagsMustache string + +//go:embed error.mustache +var contentErrorMustache string + +//go:embed base.css +var contentBaseCSS string + +//go:embed newtoc.zettel +var contentNewTOCZettel string + +//go:embed home.zettel +var contentHomeZettel string ADDED place/constplace/context.mustache Index: place/constplace/context.mustache ================================================================== --- /dev/null +++ place/constplace/context.mustache @@ -0,0 +1,16 @@ + ADDED place/constplace/delete.mustache Index: place/constplace/delete.mustache ================================================================== --- /dev/null +++ place/constplace/delete.mustache @@ -0,0 +1,15 @@ +
    +
    +

    Delete Zettel {{Zid}}

    +
    +

    Do you really want to delete this zettel?

    +
    +{{#MetaPairs}} +
    {{Key}}:
    {{Value}}
    +{{/MetaPairs}} +
    +
    + +
    +
    +{{end}} ADDED place/constplace/error.mustache Index: place/constplace/error.mustache ================================================================== --- /dev/null +++ place/constplace/error.mustache @@ -0,0 +1,6 @@ +
    +
    +

    {{ErrorTitle}}

    +
    +{{ErrorText}} +
    ADDED place/constplace/form.mustache Index: place/constplace/form.mustache ================================================================== --- /dev/null +++ place/constplace/form.mustache @@ -0,0 +1,38 @@ +
    +
    +

    {{Heading}}

    +
    +
    +
    + + +
    +
    +
    + + +
    + + +
    +
    + + +
    +
    + + +
    +
    +{{#IsTextContent}} + + +{{/IsTextContent}} +
    + +
    +
    ADDED place/constplace/home.zettel Index: place/constplace/home.zettel ================================================================== --- /dev/null +++ place/constplace/home.zettel @@ -0,0 +1,45 @@ +=== Thank you for using Zettelstore! + +You will find the lastest information about Zettelstore at [[https://zettelstore.de]]. +Check that website regulary for [[upgrades|https://zettelstore.de/home/doc/trunk/www/download.wiki]] to the latest version. +You should consult the [[change log|https://zettelstore.de/home/doc/trunk/www/changes.wiki]] before upgrading. +Sometimes, you have to edit some of your Zettelstore-related zettel before upgrading. +Since Zettelstore is currently in a development state, every upgrade might fix some of your problems. +To check for versions, there is a zettel with the [[current version|00000000000001]] of your Zettelstore. + +If you have problems concerning Zettelstore, +do not hesitate to get in [[contact with the main developer|mailto:ds@zettelstore.de]]. + +=== Reporting errors +If you have encountered an error, please include the content of the following zettel in your mail: +* [[Zettelstore Version|00000000000001]] +* [[Zettelstore Operating System|00000000000003]] +* [[Zettelstore Startup Configuration|00000000000096]] +* [[Zettelstore Startup Values|00000000000098]] +* [[Zettelstore Runtime Configuration|00000000000100]] + +Additionally, you have to describe, what you have done before that error occurs +and what you have expected instead. +Please do not forget to include the error message, if there is one. + +Some of above Zettelstore zettel can only be retrieved if you enabled ""expert mode"". +Otherwise, only some zettel are linked. +To enable expert mode, edit the zettel [[Zettelstore Runtime Configuration|00000000000100]]: +please set the metadata value of the key ''expert-mode'' to true. +To do you, enter the string ''expert-mode:true'' inside the edit view of the metadata. + +=== Information about this zettel +This zettel is your home zettel. +It is part of the Zettelstore software itself. +Every time you click on the [[Home|//]] link in the menu bar, you will be redirected to this zettel. + +You can change the content of this zettel by clicking on ""Edit"" above. +This allows you to customize your home zettel. + +Alternatively, you can designate another zettel as your home zettel. +Edit the [[Zettelstore Runtime Configuration|00000000000100]] by adding the metadata key ''home-zettel''. +Its value is the identifier of the zettel that should act as the new home zettel. +You will find the identifier of each zettel between their ""Edit"" and the ""Info"" link above. +The identifier of this zettel is ''00010000000000''. +If you provide a wrong identifier, this zettel will be shown as the home zettel. +Take a look inside the manual for further details. ADDED place/constplace/info.mustache Index: place/constplace/info.mustache ================================================================== --- /dev/null +++ place/constplace/info.mustache @@ -0,0 +1,43 @@ +
    +
    +

    Information for Zettel {{Zid}}

    +Web +· Context +{{#CanWrite}} · Edit{{/CanWrite}} +{{#CanFolge}} · Folge{{/CanFolge}} +{{#CanCopy}} · Copy{{/CanCopy}} +{{#CanRename}}· Rename{{/CanRename}} +{{#CanDelete}}· Delete{{/CanDelete}} +
    +

    Interpreted Metadata

    +{{#MetaData}}{{/MetaData}}
    {{Key}}{{{Value}}}
    +{{#HasLinks}} +

    References

    +{{#HasLocLinks}} +

    Local

    +
      +{{#LocLinks}} +
    • {{.}}
      • +{{/LocLinks}} +
    +{{/HasLocLinks}} +{{#HasExtLinks}} +

    External

    +
      +{{#ExtLinks}} +
    • {{.}}
      • +{{/ExtLinks}} +
    +{{/HasExtLinks}} +{{/HasLinks}} +

    Parts and format

    + +{{#Matrix}} + +{{#Elements}}{{#HasURL}}{{/HasURL}}{{^HasURL}}{{/HasURL}} +{{/Elements}} + +{{/Matrix}} +
    {{Text}}{{Text}}
    +{{#Endnotes}}{{{Endnotes}}}{{/Endnotes}} +
    ADDED place/constplace/listroles.mustache Index: place/constplace/listroles.mustache ================================================================== --- /dev/null +++ place/constplace/listroles.mustache @@ -0,0 +1,8 @@ + ADDED place/constplace/listtags.mustache Index: place/constplace/listtags.mustache ================================================================== --- /dev/null +++ place/constplace/listtags.mustache @@ -0,0 +1,10 @@ + ADDED place/constplace/listzettel.mustache Index: place/constplace/listzettel.mustache ================================================================== --- /dev/null +++ place/constplace/listzettel.mustache @@ -0,0 +1,19 @@ + ADDED place/constplace/login.mustache Index: place/constplace/login.mustache ================================================================== --- /dev/null +++ place/constplace/login.mustache @@ -0,0 +1,19 @@ +
    +
    +

    {{Title}}

    +
    +{{#Retry}} +
    Wrong user name / password. Try again.
    +{{/Retry}} +
    +
    + + +
    +
    + + +
    + +
    +
    ADDED place/constplace/newtoc.zettel Index: place/constplace/newtoc.zettel ================================================================== --- /dev/null +++ place/constplace/newtoc.zettel @@ -0,0 +1,4 @@ +This zettel lists all zettel that should act as a template for new zettel. +These zettel will be included in the ""New"" menu of the WebUI. +* [[New Zettel|00000000090001]] +* [[New User|00000000090002]] ADDED place/constplace/rename.mustache Index: place/constplace/rename.mustache ================================================================== --- /dev/null +++ place/constplace/rename.mustache @@ -0,0 +1,19 @@ +
    +
    +

    Rename Zettel {{.Zid}}

    +
    +

    Do you really want to rename this zettel?

    +
    +
    + + +
    + + +
    +
    +{{#MetaPairs}} +
    {{Key}}:
    {{Value}}
    +{{/MetaPairs}} +
    +
    ADDED place/constplace/zettel.mustache Index: place/constplace/zettel.mustache ================================================================== --- /dev/null +++ place/constplace/zettel.mustache @@ -0,0 +1,28 @@ +
    +
    +

    {{{HTMLTitle}}}

    +
    +{{#CanWrite}}Edit ·{{/CanWrite}} +{{Zid}} · +Info · +({{RoleText}}) +{{#HasTags}}· {{#Tags}} {{Text}}{{/Tags}}{{/HasTags}} +{{#CanCopy}}· Copy{{/CanCopy}} +{{#CanFolge}}· Folge{{/CanFolge}} +{{#FolgeRefs}}
    Folge: {{{FolgeRefs}}}{{/FolgeRefs}} +{{#PrecursorRefs}}
    Precursor: {{{PrecursorRefs}}}{{/PrecursorRefs}} +{{#HasExtURL}}
    URL: {{ExtURL}}{{/HasExtURL}} +
    +
    +{{{Content}}} +{{#HasBackLinks}} +
    +Additional links to this zettel +
      +{{#BackLinks}} +
    • {{Text}}
      • +{{/BackLinks}} +
    +
    +{{/HasBackLinks}} +
    Index: place/dirplace/directory/directory.go ================================================================== --- place/dirplace/directory/directory.go +++ place/dirplace/directory/directory.go @@ -13,24 +13,24 @@ import ( "time" "zettelstore.de/z/domain/id" - "zettelstore.de/z/place" + "zettelstore.de/z/place/change" ) // Service specifies a directory scan service. type Service struct { dirPath string rescanTime time.Duration done chan struct{} cmds chan dirCmd - infos chan<- place.ChangeInfo + infos chan<- change.Info } // NewService creates a new directory service. -func NewService(directoryPath string, rescanTime time.Duration, chci chan<- place.ChangeInfo) *Service { +func NewService(directoryPath string, rescanTime time.Duration, chci chan<- change.Info) *Service { srv := &Service{ dirPath: directoryPath, rescanTime: rescanTime, cmds: make(chan dirCmd), infos: chci, @@ -61,13 +61,13 @@ func (srv *Service) Stop() { close(srv.done) srv.done = nil } -func (srv *Service) notifyChange(reason place.ChangeReason, zid id.Zid) { +func (srv *Service) notifyChange(reason change.Reason, zid id.Zid) { if chci := srv.infos; chci != nil { - chci <- place.ChangeInfo{Reason: reason, Zid: zid} + chci <- change.Info{Reason: reason, Zid: zid} } } // NumEntries returns the number of managed zettel. func (srv *Service) NumEntries() int { Index: place/dirplace/directory/entry.go ================================================================== --- place/dirplace/directory/entry.go +++ place/dirplace/directory/entry.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -9,16 +9,11 @@ //----------------------------------------------------------------------------- // Package directory manages the directory part of a dirstore. package directory -import ( - "strings" - - "zettelstore.de/z/domain/id" - "zettelstore.de/z/domain/meta" -) +import "zettelstore.de/z/domain/id" // MetaSpec defines all possibilities where meta data can be stored. type MetaSpec int // Constants for MetaSpec @@ -41,25 +36,5 @@ // IsValid checks whether the entry is valid. func (e *Entry) IsValid() bool { return e.Zid.IsValid() } - -var alternativeSyntax = map[string]string{ - "htm": "html", -} - -func (e *Entry) calculateSyntax() string { - ext := strings.ToLower(e.ContentExt) - if syntax, ok := alternativeSyntax[ext]; ok { - return syntax - } - return ext -} - -// CalcDefaultMeta returns metadata with default values for the given entry. -func (e *Entry) CalcDefaultMeta() *meta.Meta { - m := meta.New(e.Zid) - m.Set(meta.KeyTitle, e.Zid.String()) - m.Set(meta.KeySyntax, e.calculateSyntax()) - return m -} Index: place/dirplace/directory/service.go ================================================================== --- place/dirplace/directory/service.go +++ place/dirplace/directory/service.go @@ -15,10 +15,11 @@ "log" "time" "zettelstore.de/z/domain/id" "zettelstore.de/z/place" + "zettelstore.de/z/place/change" ) // ping sends every tick a signal to reload the directory list func ping(tick chan<- struct{}, rescanTime time.Duration, done <-chan struct{}) { ticker := time.NewTicker(rescanTime) @@ -109,11 +110,11 @@ if ready != nil { ready <- len(curMap) close(ready) ready = nil } - srv.notifyChange(place.OnReload, id.Invalid) + srv.notifyChange(change.OnReload, id.Invalid) case fileStatusError: log.Println("DIRPLACE", "ERROR", ev.err) case fileStatusUpdate: srv.processFileUpdateEvent(ev, curMap, newMap) case fileStatusDelete: @@ -130,20 +131,20 @@ func (srv *Service) processFileUpdateEvent(ev *fileEvent, curMap, newMap dirMap) { if newMap != nil { dirMapUpdate(newMap, ev) } else { dirMapUpdate(curMap, ev) - srv.notifyChange(place.OnUpdate, ev.zid) + srv.notifyChange(change.OnUpdate, ev.zid) } } func (srv *Service) processFileDeleteEvent(ev *fileEvent, curMap, newMap dirMap) { if newMap != nil { deleteFromMap(newMap, ev) } else { deleteFromMap(curMap, ev) - srv.notifyChange(place.OnDelete, ev.zid) + srv.notifyChange(change.OnDelete, ev.zid) } } type dirCmd interface { run(m dirMap) Index: place/dirplace/directory/watch.go ================================================================== --- place/dirplace/directory/watch.go +++ place/dirplace/directory/watch.go @@ -10,11 +10,10 @@ // Package directory manages the directory part of a directory place. package directory import ( - "io/ioutil" "os" "path/filepath" "regexp" "time" @@ -97,11 +96,11 @@ } reloadStartEvent := &fileEvent{status: fileStatusReloadStart} reloadEndEvent := &fileEvent{status: fileStatusReloadEnd} reloadFiles := func() bool { - files, err := ioutil.ReadDir(directory) + entries, err := os.ReadDir(directory) if err != nil { if res := sendError(err); res != sendDone { return res == sendReload } return true @@ -119,15 +118,18 @@ if res := sendError(err); res != sendDone { return res == sendReload } } - for _, file := range files { - if !file.Mode().IsRegular() { + for _, entry := range entries { + if entry.IsDir() { + continue + } + if info, err1 := entry.Info(); err1 != nil || !info.Mode().IsRegular() { continue } - name := file.Name() + name := entry.Name() match := matchValidFileName(name) if len(match) > 0 { path := filepath.Join(directory, name) if res := sendFileEvent(fileStatusUpdate, path, match); res != sendDone { return res == sendReload Index: place/dirplace/dirplace.go ================================================================== --- place/dirplace/dirplace.go +++ place/dirplace/dirplace.go @@ -25,15 +25,17 @@ "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/place" "zettelstore.de/z/place/dirplace/directory" + "zettelstore.de/z/place/fileplace" "zettelstore.de/z/place/manager" + "zettelstore.de/z/search" ) func init() { - manager.Register("dir", func(u *url.URL, cdata *manager.ConnectData) (place.Place, error) { + manager.Register("dir", func(u *url.URL, cdata *manager.ConnectData) (place.ManagedPlace, error) { path := getDirPath(u) if _, err := os.Stat(path); os.IsNotExist(err) { return nil, err } dp := dirPlace{ @@ -134,12 +136,11 @@ func (dp *dirPlace) CanCreateZettel(ctx context.Context) bool { return !dp.readonly } -func (dp *dirPlace) CreateZettel( - ctx context.Context, zettel domain.Zettel) (id.Zid, error) { +func (dp *dirPlace) CreateZettel(ctx context.Context, zettel domain.Zettel) (id.Zid, error) { if dp.readonly { return id.Invalid, place.ErrReadOnly } meta := zettel.Meta @@ -188,34 +189,31 @@ result[entry.Zid] = true } return result, nil } -func (dp *dirPlace) SelectMeta( - ctx context.Context, f *place.Filter, s *place.Sorter) (res []*meta.Meta, err error) { - - hasMatch := place.CreateFilterFunc(f) +func (dp *dirPlace) SelectMeta(ctx context.Context, match search.MetaMatchFunc) (res []*meta.Meta, err error) { entries := dp.dirSrv.GetEntries() res = make([]*meta.Meta, 0, len(entries)) + // The following loop could be parallelized if needed for performance. for _, entry := range entries { - // TODO: execute requests in parallel m, err1 := getMeta(dp, &entry, entry.Zid) err = err1 if err != nil { continue } dp.cleanupMeta(ctx, m) dp.cdata.Filter.Enrich(ctx, m) - if hasMatch(m) { + if match(m) { res = append(res, m) } } if err != nil { return nil, err } - return place.ApplySorter(res, s), nil + return res, nil } func (dp *dirPlace) CanUpdateZettel(ctx context.Context, zettel domain.Zettel) bool { return !dp.readonly } @@ -233,11 +231,12 @@ if !entry.IsValid() { // Existing zettel, but new in this place. entry.Zid = meta.Zid dp.updateEntryFromMeta(&entry, meta) } else if entry.MetaSpec == directory.MetaSpecNone { - if defaultMeta := entry.CalcDefaultMeta(); !meta.Equal(defaultMeta, true) { + defaultMeta := fileplace.CalcDefaultMeta(entry.Zid, entry.ContentExt) + if !meta.Equal(defaultMeta, true) { dp.updateEntryFromMeta(&entry, meta) dp.dirSrv.UpdateEntry(&entry) } } return setZettel(dp, &entry, zettel) Index: place/dirplace/service.go ================================================================== --- place/dirplace/service.go +++ place/dirplace/service.go @@ -10,18 +10,18 @@ // Package dirplace provides a directory-based zettel place. package dirplace import ( - "io/ioutil" "os" "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/input" "zettelstore.de/z/place/dirplace/directory" + "zettelstore.de/z/place/fileplace" ) func fileService(num uint32, cmds <-chan fileCmd) { for cmd := range cmds { cmd.run() @@ -52,22 +52,23 @@ meta *meta.Meta err error } func (cmd *fileGetMeta) run() { + entry := cmd.entry var m *meta.Meta var err error - switch cmd.entry.MetaSpec { + switch entry.MetaSpec { case directory.MetaSpecFile: - m, err = parseMetaFile(cmd.entry.Zid, cmd.entry.MetaPath) + m, err = parseMetaFile(entry.Zid, entry.MetaPath) case directory.MetaSpecHeader: - m, _, err = parseMetaContentFile(cmd.entry.Zid, cmd.entry.ContentPath) + m, _, err = parseMetaContentFile(entry.Zid, entry.ContentPath) default: - m = cmd.entry.CalcDefaultMeta() + m = fileplace.CalcDefaultMeta(entry.Zid, entry.ContentExt) } if err == nil { - cleanupMeta(m, cmd.entry) + cmdCleanupMeta(m, entry) } cmd.rc <- resGetMeta{m, err} } // COMMAND: getMetaContent ---------------------------------------- @@ -95,24 +96,25 @@ func (cmd *fileGetMetaContent) run() { var m *meta.Meta var content string var err error - switch cmd.entry.MetaSpec { + entry := cmd.entry + switch entry.MetaSpec { case directory.MetaSpecFile: - m, err = parseMetaFile(cmd.entry.Zid, cmd.entry.MetaPath) + m, err = parseMetaFile(entry.Zid, entry.MetaPath) if err == nil { - content, err = readFileContent(cmd.entry.ContentPath) + content, err = readFileContent(entry.ContentPath) } case directory.MetaSpecHeader: - m, content, err = parseMetaContentFile(cmd.entry.Zid, cmd.entry.ContentPath) + m, content, err = parseMetaContentFile(entry.Zid, entry.ContentPath) default: - m = cmd.entry.CalcDefaultMeta() - content, err = readFileContent(cmd.entry.ContentPath) + m = fileplace.CalcDefaultMeta(entry.Zid, entry.ContentExt) + content, err = readFileContent(entry.ContentPath) } if err == nil { - cleanupMeta(m, cmd.entry) + cmdCleanupMeta(m, entry) } cmd.rc <- resGetMetaContent{m, content, err} } // COMMAND: setZettel ---------------------------------------- @@ -224,11 +226,11 @@ } // Utility functions ---------------------------------------- func readFileContent(path string) (string, error) { - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { return "", err } return string(data), nil } @@ -250,29 +252,17 @@ inp := input.NewInput(src) meta := meta.NewFromInput(zid, inp) return meta, src[inp.Pos:], nil } -func cleanupMeta(m *meta.Meta, entry *directory.Entry) { - if title, ok := m.Get(meta.KeyTitle); !ok || title == "" { - m.Set(meta.KeyTitle, entry.Zid.String()) - } - - if entry.MetaSpec == directory.MetaSpecFile { - if syntax, ok := m.Get(meta.KeySyntax); !ok || syntax == "" { - dm := entry.CalcDefaultMeta() - syntax, ok = dm.Get(meta.KeySyntax) - if !ok { - panic("Default meta must contain syntax") - } - m.Set(meta.KeySyntax, syntax) - } - } - - if entry.Duplicates { - m.Set(meta.KeyDuplicates, meta.ValueTrue) - } +func cmdCleanupMeta(m *meta.Meta, entry *directory.Entry) { + fileplace.CleanupMeta( + m, + entry.Zid, entry.ContentExt, + entry.MetaSpec == directory.MetaSpecFile, + entry.Duplicates, + ) } func openFileWrite(path string) (*os.File, error) { return os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) } ADDED place/fileplace/fileplace.go Index: place/fileplace/fileplace.go ================================================================== --- /dev/null +++ place/fileplace/fileplace.go @@ -0,0 +1,90 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package fileplace provides places that are stored in a file. +package fileplace + +import ( + "errors" + "net/url" + "path/filepath" + "strings" + + "zettelstore.de/z/domain/id" + "zettelstore.de/z/domain/meta" + "zettelstore.de/z/place" + "zettelstore.de/z/place/manager" +) + +func init() { + manager.Register("file", func(u *url.URL, cdata *manager.ConnectData) (place.ManagedPlace, error) { + path := getFilepathFromURL(u) + ext := strings.ToLower(filepath.Ext(path)) + if ext != ".zip" { + return nil, errors.New("unknown extension '" + ext + "' in place URL: " + u.String()) + } + return &zipPlace{name: path, filter: cdata.Filter}, nil + }) +} + +func getFilepathFromURL(u *url.URL) string { + name := u.Opaque + if name == "" { + name = u.Path + } + components := strings.Split(name, "/") + fileName := filepath.Join(components...) + if len(components) > 0 && components[0] == "" { + return "/" + fileName + } + return fileName +} + +var alternativeSyntax = map[string]string{ + "htm": "html", +} + +func calculateSyntax(ext string) string { + ext = strings.ToLower(ext) + if syntax, ok := alternativeSyntax[ext]; ok { + return syntax + } + return ext +} + +// CalcDefaultMeta returns metadata with default values for the given entry. +func CalcDefaultMeta(zid id.Zid, ext string) *meta.Meta { + m := meta.New(zid) + m.Set(meta.KeyTitle, zid.String()) + m.Set(meta.KeySyntax, calculateSyntax(ext)) + return m +} + +// CleanupMeta enhances the given metadata. +func CleanupMeta(m *meta.Meta, zid id.Zid, ext string, inMeta, duplicates bool) { + if title, ok := m.Get(meta.KeyTitle); !ok || title == "" { + m.Set(meta.KeyTitle, zid.String()) + } + + if inMeta { + if syntax, ok := m.Get(meta.KeySyntax); !ok || syntax == "" { + dm := CalcDefaultMeta(zid, ext) + syntax, ok = dm.Get(meta.KeySyntax) + if !ok { + panic("Default meta must contain syntax") + } + m.Set(meta.KeySyntax, syntax) + } + } + + if duplicates { + m.Set(meta.KeyDuplicates, meta.ValueTrue) + } +} ADDED place/fileplace/zipplace.go Index: place/fileplace/zipplace.go ================================================================== --- /dev/null +++ place/fileplace/zipplace.go @@ -0,0 +1,261 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package fileplace provides places that are stored in a file. +package fileplace + +import ( + "archive/zip" + "context" + "io" + "regexp" + "strings" + + "zettelstore.de/z/domain" + "zettelstore.de/z/domain/id" + "zettelstore.de/z/domain/meta" + "zettelstore.de/z/index" + "zettelstore.de/z/input" + "zettelstore.de/z/place" + "zettelstore.de/z/search" +) + +var validFileName = regexp.MustCompile(`^(\d{14}).*(\.(.+))$`) + +func matchValidFileName(name string) []string { + return validFileName.FindStringSubmatch(name) +} + +type zipEntry struct { + metaName string + contentName string + contentExt string // (normalized) file extension of zettel content + metaInHeader bool +} + +type zipPlace struct { + name string + filter index.MetaFilter + zettel map[id.Zid]*zipEntry // no lock needed, because read-only after creation +} + +func (zp *zipPlace) Location() string { + if strings.HasPrefix(zp.name, "/") { + return "file://" + zp.name + } + return "file:" + zp.name +} + +func (zp *zipPlace) Start(ctx context.Context) error { + reader, err := zip.OpenReader(zp.name) + if err != nil { + return err + } + defer reader.Close() + zp.zettel = make(map[id.Zid]*zipEntry) + for _, f := range reader.File { + match := matchValidFileName(f.Name) + if len(match) < 1 { + continue + } + zid, err := id.Parse(match[1]) + if err != nil { + continue + } + zp.addFile(zid, f.Name, match[3]) + } + return nil +} + +func (zp *zipPlace) addFile(zid id.Zid, name, ext string) { + entry := zp.zettel[zid] + if entry == nil { + entry = &zipEntry{} + zp.zettel[zid] = entry + } + switch ext { + case "zettel": + if entry.contentExt == "" { + entry.contentName = name + entry.contentExt = ext + entry.metaInHeader = true + } + case "meta": + entry.metaName = name + entry.metaInHeader = false + default: + if entry.contentExt == "" { + entry.contentExt = ext + entry.contentName = name + } + } +} + +func (zp *zipPlace) Stop(ctx context.Context) error { + zp.zettel = nil + return nil +} + +func (zp *zipPlace) CanCreateZettel(ctx context.Context) bool { return false } + +func (zp *zipPlace) CreateZettel(ctx context.Context, zettel domain.Zettel) (id.Zid, error) { + return id.Invalid, place.ErrReadOnly +} + +func (zp *zipPlace) GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error) { + entry, ok := zp.zettel[zid] + if !ok { + return domain.Zettel{}, place.ErrNotFound + } + reader, err := zip.OpenReader(zp.name) + if err != nil { + return domain.Zettel{}, err + } + defer reader.Close() + + var m *meta.Meta + var src string + var inMeta bool + if entry.metaInHeader { + src, err = readZipFileContent(reader, entry.contentName) + if err != nil { + return domain.Zettel{}, err + } + inp := input.NewInput(src) + m = meta.NewFromInput(zid, inp) + src = src[inp.Pos:] + } else if metaName := entry.metaName; metaName != "" { + m, err = readZipMetaFile(reader, zid, metaName) + if err != nil { + return domain.Zettel{}, err + } + src, err = readZipFileContent(reader, entry.contentName) + if err != nil { + return domain.Zettel{}, err + } + inMeta = true + } else { + m = CalcDefaultMeta(zid, entry.contentExt) + } + CleanupMeta(m, zid, entry.contentExt, inMeta, false) + return domain.Zettel{Meta: m, Content: domain.NewContent(src)}, nil +} + +func (zp *zipPlace) GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) { + entry, ok := zp.zettel[zid] + if !ok { + return nil, place.ErrNotFound + } + reader, err := zip.OpenReader(zp.name) + if err != nil { + return nil, err + } + defer reader.Close() + return readZipMeta(reader, zid, entry) +} + +func (zp *zipPlace) FetchZids(ctx context.Context) (id.Set, error) { + result := id.NewSetCap(len(zp.zettel)) + for zid := range zp.zettel { + result[zid] = true + } + return result, nil +} + +func (zp *zipPlace) SelectMeta(ctx context.Context, match search.MetaMatchFunc) (res []*meta.Meta, err error) { + reader, err := zip.OpenReader(zp.name) + if err != nil { + return nil, err + } + defer reader.Close() + for zid, entry := range zp.zettel { + m, err := readZipMeta(reader, zid, entry) + if err != nil { + continue + } + zp.filter.Enrich(ctx, m) + if match(m) { + res = append(res, m) + } + } + return res, nil +} + +func (zp *zipPlace) CanUpdateZettel(ctx context.Context, zettel domain.Zettel) bool { + return false +} + +func (zp *zipPlace) UpdateZettel(ctx context.Context, zettel domain.Zettel) error { + return place.ErrReadOnly +} + +func (zp *zipPlace) AllowRenameZettel(ctx context.Context, zid id.Zid) bool { + _, ok := zp.zettel[zid] + return !ok +} + +func (zp *zipPlace) RenameZettel(ctx context.Context, curZid, newZid id.Zid) error { + if _, ok := zp.zettel[curZid]; ok { + return place.ErrReadOnly + } + return place.ErrNotFound +} + +func (zp *zipPlace) CanDeleteZettel(ctx context.Context, zid id.Zid) bool { return false } + +func (zp *zipPlace) DeleteZettel(ctx context.Context, zid id.Zid) error { + if _, ok := zp.zettel[zid]; ok { + return place.ErrReadOnly + } + return place.ErrNotFound +} + +func (zp *zipPlace) ReadStats(st *place.Stats) { + st.ReadOnly = true + st.Zettel = len(zp.zettel) +} + +func readZipMeta(reader *zip.ReadCloser, zid id.Zid, entry *zipEntry) (m *meta.Meta, err error) { + var inMeta bool + if entry.metaInHeader { + m, err = readZipMetaFile(reader, zid, entry.contentName) + } else if metaName := entry.metaName; metaName != "" { + m, err = readZipMetaFile(reader, zid, entry.metaName) + inMeta = true + } else { + m = CalcDefaultMeta(zid, entry.contentExt) + } + if err == nil { + CleanupMeta(m, zid, entry.contentExt, inMeta, false) + } + return m, err +} + +func readZipMetaFile(reader *zip.ReadCloser, zid id.Zid, name string) (*meta.Meta, error) { + src, err := readZipFileContent(reader, name) + if err != nil { + return nil, err + } + inp := input.NewInput(src) + return meta.NewFromInput(zid, inp), nil +} + +func readZipFileContent(reader *zip.ReadCloser, name string) (string, error) { + f, err := reader.Open(name) + if err != nil { + return "", err + } + defer f.Close() + buf, err := io.ReadAll(f) + if err != nil { + return "", err + } + return string(buf), nil +} DELETED place/filter.go Index: place/filter.go ================================================================== --- place/filter.go +++ /dev/null @@ -1,318 +0,0 @@ -//----------------------------------------------------------------------------- -// Copyright (c) 2020-2021 Detlef Stern -// -// This file is part of zettelstore. -// -// Zettelstore is licensed under the latest version of the EUPL (European Union -// Public License). Please see file LICENSE.txt for your rights and obligations -// under this license. -//----------------------------------------------------------------------------- - -// Package place provides a generic interface to zettel places. -package place - -import ( - "strings" - - "zettelstore.de/z/domain/meta" -) - -// EnsureFilter make sure that there is a current filter. -func EnsureFilter(filter *Filter) *Filter { - if filter == nil { - filter = new(Filter) - filter.Expr = make(FilterExpr) - } - return filter -} - -// FilterFunc is a predicate to check if given meta must be selected. -type FilterFunc func(*meta.Meta) bool - -func selectAll(m *meta.Meta) bool { return true } - -type matchFunc func(value string) bool - -func matchNever(value string) bool { return false } - -type matchSpec struct { - key string - match matchFunc -} - -// CreateFilterFunc calculates a filter func based on the given filter. -func CreateFilterFunc(filter *Filter) FilterFunc { - if filter == nil { - return selectAll - } - specs, searchAll := createFilterSpecs(filter) - if len(specs) == 0 { - if searchAll == nil { - if sel := filter.Select; sel != nil { - return sel - } - return selectAll - } - return addSelectFunc(filter, searchAll) - } - negate := filter.Negate - searchMeta := func(m *meta.Meta) bool { - for _, s := range specs { - value, ok := m.Get(s.key) - if !ok || !s.match(value) { - return negate - } - } - return !negate - } - if searchAll == nil { - return addSelectFunc(filter, searchMeta) - } - return addSelectFunc(filter, func(meta *meta.Meta) bool { - return searchAll(meta) || searchMeta(meta) - }) -} - -func createFilterSpecs(filter *Filter) ([]matchSpec, FilterFunc) { - specs := make([]matchSpec, 0, len(filter.Expr)) - var searchAll FilterFunc - for key, values := range filter.Expr { - if key == "" { - // Special handling if searching all keys... - searchAll = createSearchAllFunc(values, filter.Negate) - continue - } - if meta.KeyIsValid(key) { - match := createMatchFunc(key, values) - if match != nil { - specs = append(specs, matchSpec{key, match}) - } - } - } - return specs, searchAll -} - -func addSelectFunc(filter *Filter, f FilterFunc) FilterFunc { - if filter == nil { - return f - } - if sel := filter.Select; sel != nil { - return func(meta *meta.Meta) bool { - return sel(meta) && f(meta) - } - } - return f -} - -func createMatchFunc(key string, values []string) matchFunc { - switch meta.Type(key) { - case meta.TypeBool: - return createMatchBoolFunc(values) - case meta.TypeCredential: - return matchNever - case meta.TypeID, meta.TypeTimestamp: // ID and timestamp use the same layout - return createMatchIDFunc(values) - case meta.TypeIDSet: - return createMatchIDSetFunc(values) - case meta.TypeTagSet: - return createMatchTagSetFunc(values) - case meta.TypeWord: - return createMatchWordFunc(values) - case meta.TypeWordSet: - return createMatchWordSetFunc(values) - } - return createMatchStringFunc(values) -} - -func createMatchBoolFunc(values []string) matchFunc { - preValues := make([]bool, 0, len(values)) - for _, v := range values { - preValues = append(preValues, meta.BoolValue(v)) - } - return func(value string) bool { - bValue := meta.BoolValue(value) - for _, v := range preValues { - if bValue != v { - return false - } - } - return true - } -} - -func createMatchIDFunc(values []string) matchFunc { - return func(value string) bool { - for _, v := range values { - if !strings.HasPrefix(value, v) { - return false - } - } - return true - } -} - -func createMatchIDSetFunc(values []string) matchFunc { - idValues := preprocessSet(sliceToLower(values)) - return func(value string) bool { - ids := meta.ListFromValue(value) - for _, neededIDs := range idValues { - for _, neededID := range neededIDs { - if !matchAllID(ids, neededID) { - return false - } - } - } - return true - } -} - -func createMatchTagSetFunc(values []string) matchFunc { - tagValues := preprocessSet(values) - return func(value string) bool { - tags := meta.ListFromValue(value) - // Remove leading '#' from each tag - for i, tag := range tags { - tags[i] = meta.CleanTag(tag) - } - for _, neededTags := range tagValues { - for _, neededTag := range neededTags { - if !matchAllTag(tags, neededTag) { - return false - } - } - } - return true - } -} - -func createMatchWordFunc(values []string) matchFunc { - values = sliceToLower(values) - return func(value string) bool { - value = strings.ToLower(value) - for _, v := range values { - if value != v { - return false - } - } - return true - } -} - -func createMatchWordSetFunc(values []string) matchFunc { - wordValues := preprocessSet(sliceToLower(values)) - return func(value string) bool { - words := meta.ListFromValue(value) - for _, neededWords := range wordValues { - for _, neededWord := range neededWords { - if !matchAllWord(words, neededWord) { - return false - } - } - } - return true - } -} - -func createMatchStringFunc(values []string) matchFunc { - values = sliceToLower(values) - return func(value string) bool { - value = strings.ToLower(value) - for _, v := range values { - if !strings.Contains(value, v) { - return false - } - } - return true - } -} - -func createSearchAllFunc(values []string, negate bool) FilterFunc { - matchFuncs := map[*meta.DescriptionType]matchFunc{} - return func(m *meta.Meta) bool { - for _, p := range m.Pairs(true) { - keyType := meta.Type(p.Key) - match, ok := matchFuncs[keyType] - if !ok { - if keyType == meta.TypeBool { - match = createBoolSearchFunc(p.Key, values) - } else { - match = createMatchFunc(p.Key, values) - } - matchFuncs[keyType] = match - } - if match(p.Value) { - return !negate - } - } - match, ok := matchFuncs[meta.Type(meta.KeyID)] - if !ok { - match = createMatchFunc(meta.KeyID, values) - } - return match(m.Zid.String()) != negate - } -} - -// createBoolSearchFunc only creates a matchFunc if the values to compare are -// possible bool values. Otherwise every meta with a bool key could match the -// search query. -func createBoolSearchFunc(key string, values []string) matchFunc { - for _, v := range values { - if len(v) > 0 && !strings.ContainsRune("01tfTFynYN", rune(v[0])) { - return func(value string) bool { return false } - } - } - return createMatchFunc(key, values) -} - -func sliceToLower(sl []string) []string { - result := make([]string, 0, len(sl)) - for _, s := range sl { - result = append(result, strings.ToLower(s)) - } - return result -} - -func preprocessSet(set []string) [][]string { - result := make([][]string, 0, len(set)) - for _, elem := range set { - splitElems := strings.Split(elem, ",") - valueElems := make([]string, 0, len(splitElems)) - for _, se := range splitElems { - e := strings.TrimSpace(se) - if len(e) > 0 { - valueElems = append(valueElems, e) - } - } - if len(valueElems) > 0 { - result = append(result, valueElems) - } - } - return result -} - -func matchAllID(zettelIDs []string, neededID string) bool { - for _, zt := range zettelIDs { - if strings.HasPrefix(zt, neededID) { - return true - } - } - return false -} - -func matchAllTag(zettelTags []string, neededTag string) bool { - for _, zt := range zettelTags { - if zt == neededTag { - return true - } - } - return false -} - -func matchAllWord(zettelWords []string, neededWord string) bool { - for _, zw := range zettelWords { - if zw == neededWord { - return true - } - } - return false -} Index: place/manager/manager.go ================================================================== --- place/manager/manager.go +++ place/manager/manager.go @@ -13,29 +13,33 @@ import ( "context" "log" "net/url" + "runtime/debug" "sort" "strings" "sync" + "zettelstore.de/z/config/startup" "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/index" "zettelstore.de/z/place" + "zettelstore.de/z/place/change" + "zettelstore.de/z/search" ) // ConnectData contains all administration related values. type ConnectData struct { Filter index.MetaFilter - Notify chan<- place.ChangeInfo + Notify chan<- change.Info } // Connect returns a handle to the specified place -func Connect(rawURL string, readonlyMode bool, cdata *ConnectData) (place.Place, error) { +func Connect(rawURL string, readonlyMode bool, cdata *ConnectData) (place.ManagedPlace, error) { u, err := url.Parse(rawURL) if err != nil { return nil, err } if u.Scheme == "" { @@ -63,11 +67,11 @@ // ErrInvalidScheme is returned if there is no place with the given scheme type ErrInvalidScheme struct{ Scheme string } func (err *ErrInvalidScheme) Error() string { return "Invalid scheme: " + err.Scheme } -type createFunc func(*url.URL, *ConnectData) (place.Place, error) +type createFunc func(*url.URL, *ConnectData) (place.ManagedPlace, error) var registry = map[string]createFunc{} // Register the encoder for later retrieval. func Register(scheme string, create createFunc) { @@ -89,32 +93,34 @@ // Manager is a coordinating place. type Manager struct { mx sync.RWMutex started bool - subplaces []place.Place + subplaces []place.ManagedPlace filter index.MetaFilter - observers []func(place.ChangeInfo) + observers []change.Func mxObserver sync.RWMutex done chan struct{} - infos chan place.ChangeInfo + infos chan change.Info } // New creates a new managing place. func New(placeURIs []string, readonlyMode bool, filter index.MetaFilter) (*Manager, error) { mgr := &Manager{ filter: filter, - infos: make(chan place.ChangeInfo, len(placeURIs)*10), + infos: make(chan change.Info, len(placeURIs)*10), } cdata := ConnectData{Filter: filter, Notify: mgr.infos} - subplaces := make([]place.Place, 0, len(placeURIs)+2) + subplaces := make([]place.ManagedPlace, 0, len(placeURIs)+2) for _, uri := range placeURIs { p, err := Connect(uri, readonlyMode, &cdata) if err != nil { return nil, err } - subplaces = append(subplaces, p) + if p != nil { + subplaces = append(subplaces, p) + } } constplace, err := registry[" const"](nil, &cdata) if err != nil { return nil, err } @@ -127,31 +133,33 @@ return mgr, nil } // RegisterObserver registers an observer that will be notified // if a zettel was found to be changed. -func (mgr *Manager) RegisterObserver(f func(place.ChangeInfo)) { +func (mgr *Manager) RegisterObserver(f change.Func) { if f != nil { mgr.mxObserver.Lock() mgr.observers = append(mgr.observers, f) mgr.mxObserver.Unlock() } } -func (mgr *Manager) notifyObserver(ci place.ChangeInfo) { +func (mgr *Manager) notifyObserver(ci change.Info) { mgr.mxObserver.RLock() observers := mgr.observers mgr.mxObserver.RUnlock() for _, ob := range observers { ob(ci) } } -func notifier(notify func(place.ChangeInfo), infos <-chan place.ChangeInfo, done <-chan struct{}) { +func notifier(notify change.Func, infos <-chan change.Info, done <-chan struct{}) { // The call to notify may panic. Ensure a running notifier. defer func() { - if err := recover(); err != nil { + if r := recover(); r != nil { + log.Println("recovered from:", r) + debug.PrintStack() go notifier(notify, infos, done) } }() for { @@ -210,11 +218,11 @@ } mgr.done = make(chan struct{}) go notifier(mgr.notifyObserver, mgr.infos, mgr.done) mgr.started = true mgr.mx.Unlock() - mgr.infos <- place.ChangeInfo{Reason: place.OnReload, Zid: id.Invalid} + mgr.infos <- change.Info{Reason: change.OnReload, Zid: id.Invalid} return nil } // Stop the started place. Now only the Start() function is allowed. func (mgr *Manager) Stop(ctx context.Context) error { @@ -261,11 +269,13 @@ if !mgr.started { return domain.Zettel{}, place.ErrStopped } for _, p := range mgr.subplaces { if z, err := p.GetZettel(ctx, zid); err != place.ErrNotFound { - mgr.filter.Enrich(ctx, z.Meta) + if err == nil { + mgr.filter.Enrich(ctx, z.Meta) + } return z, err } } return domain.Zettel{}, place.ErrNotFound } @@ -277,11 +287,13 @@ if !mgr.started { return nil, place.ErrStopped } for _, p := range mgr.subplaces { if m, err := p.GetMeta(ctx, zid); err != place.ErrNotFound { - mgr.filter.Enrich(ctx, m) + if err == nil { + mgr.filter.Enrich(ctx, m) + } return m, err } } return nil, place.ErrNotFound } @@ -314,32 +326,34 @@ return result, nil } // SelectMeta returns all zettel meta data that match the selection // criteria. The result is ordered by descending zettel id. -func (mgr *Manager) SelectMeta(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) { +func (mgr *Manager) SelectMeta(ctx context.Context, s *search.Search) ([]*meta.Meta, error) { mgr.mx.RLock() defer mgr.mx.RUnlock() if !mgr.started { return nil, place.ErrStopped } var result []*meta.Meta + match := s.CompileMatch(startup.Indexer()) for _, p := range mgr.subplaces { - selected, err := p.SelectMeta(ctx, f, nil) + selected, err := p.SelectMeta(ctx, match) if err != nil { return nil, err } + sort.Slice(selected, func(i, j int) bool { return selected[i].Zid > selected[j].Zid }) if len(result) == 0 { result = selected } else { result = place.MergeSorted(result, selected) } } if s == nil { return result, nil } - return place.ApplySorter(result, s), nil + return s.Sort(result), nil } // CanUpdateZettel returns true, if place could possibly update the given zettel. func (mgr *Manager) CanUpdateZettel(ctx context.Context, zettel domain.Zettel) bool { mgr.mx.RLock() Index: place/memplace/memplace.go ================================================================== --- place/memplace/memplace.go +++ place/memplace/memplace.go @@ -19,17 +19,19 @@ "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/place" + "zettelstore.de/z/place/change" "zettelstore.de/z/place/manager" + "zettelstore.de/z/search" ) func init() { manager.Register( "mem", - func(u *url.URL, cdata *manager.ConnectData) (place.Place, error) { + func(u *url.URL, cdata *manager.ConnectData) (place.ManagedPlace, error) { return &memPlace{u: u, cdata: *cdata}, nil }) } type memPlace struct { @@ -37,13 +39,13 @@ cdata manager.ConnectData zettel map[id.Zid]domain.Zettel mx sync.RWMutex } -func (mp *memPlace) notifyChanged(reason place.ChangeReason, zid id.Zid) { +func (mp *memPlace) notifyChanged(reason change.Reason, zid id.Zid) { if chci := mp.cdata.Notify; chci != nil { - chci <- place.ChangeInfo{Reason: reason, Zid: zid} + chci <- change.Info{Reason: reason, Zid: zid} } } func (mp *memPlace) Location() string { return mp.u.String() @@ -70,11 +72,11 @@ meta := zettel.Meta.Clone() meta.Zid = mp.calcNewZid() zettel.Meta = meta mp.zettel[meta.Zid] = zettel mp.mx.Unlock() - mp.notifyChanged(place.OnUpdate, meta.Zid) + mp.notifyChanged(change.OnUpdate, meta.Zid) return meta.Zid, nil } func (mp *memPlace) calcNewZid() id.Zid { zid := id.New(false) @@ -119,23 +121,22 @@ } mp.mx.RUnlock() return result, nil } -func (mp *memPlace) SelectMeta(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) { - filterFunc := place.CreateFilterFunc(f) +func (mp *memPlace) SelectMeta(ctx context.Context, match search.MetaMatchFunc) ([]*meta.Meta, error) { result := make([]*meta.Meta, 0, len(mp.zettel)) mp.mx.RLock() for _, zettel := range mp.zettel { m := zettel.Meta.Clone() mp.cdata.Filter.Enrich(ctx, m) - if filterFunc(m) { + if match(m) { result = append(result, m) } } mp.mx.RUnlock() - return place.ApplySorter(result, s), nil + return result, nil } func (mp *memPlace) CanUpdateZettel(ctx context.Context, zettel domain.Zettel) bool { return true } @@ -147,11 +148,11 @@ return &place.ErrInvalidID{Zid: meta.Zid} } zettel.Meta = meta mp.zettel[meta.Zid] = zettel mp.mx.Unlock() - mp.notifyChanged(place.OnUpdate, meta.Zid) + mp.notifyChanged(change.OnUpdate, meta.Zid) return nil } func (mp *memPlace) AllowRenameZettel(ctx context.Context, zid id.Zid) bool { return true } @@ -173,12 +174,12 @@ meta.Zid = newZid zettel.Meta = meta mp.zettel[newZid] = zettel delete(mp.zettel, curZid) mp.mx.Unlock() - mp.notifyChanged(place.OnDelete, curZid) - mp.notifyChanged(place.OnUpdate, newZid) + mp.notifyChanged(change.OnDelete, curZid) + mp.notifyChanged(change.OnUpdate, newZid) return nil } func (mp *memPlace) CanDeleteZettel(ctx context.Context, zid id.Zid) bool { mp.mx.RLock() @@ -193,15 +194,15 @@ mp.mx.Unlock() return place.ErrNotFound } delete(mp.zettel, zid) mp.mx.Unlock() - mp.notifyChanged(place.OnDelete, zid) + mp.notifyChanged(change.OnDelete, zid) return nil } func (mp *memPlace) ReadStats(st *place.Stats) { st.ReadOnly = false mp.mx.RLock() st.Zettel = len(mp.zettel) mp.mx.RUnlock() } Index: place/merge.go ================================================================== --- place/merge.go +++ place/merge.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -11,11 +11,11 @@ // Package place provides a generic interface to zettel places. package place import "zettelstore.de/z/domain/meta" -// MergeSorted returns a merged sequence of meta data, sorted by a given Sorter. +// MergeSorted returns a merged sequence of metadata, sorted by Zid. // The lists first and second must be sorted descending by Zid. func MergeSorted(first, second []*meta.Meta) []*meta.Meta { lenFirst := len(first) lenSecond := len(second) result := make([]*meta.Meta, 0, lenFirst+lenSecond) Index: place/place.go ================================================================== --- place/place.go +++ place/place.go @@ -17,14 +17,16 @@ "fmt" "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" + "zettelstore.de/z/place/change" + "zettelstore.de/z/search" ) -// Place is implemented by all Zettel places. -type Place interface { +// BasePlace is implemented by all Zettel places. +type BasePlace interface { // Location returns some information where the place is located. // Format is dependent of the place. Location() string // CanCreateZettel returns true, if place could possibly create a new zettel. @@ -41,14 +43,10 @@ GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) // FetchZids returns the set of all zettel identifer managed by the place. FetchZids(ctx context.Context) (id.Set, error) - // SelectMeta returns all zettel meta data that match the selection criteria. - // TODO: more docs - SelectMeta(ctx context.Context, f *Filter, s *Sorter) ([]*meta.Meta, error) - // CanUpdateZettel returns true, if place could possibly update the given zettel. CanUpdateZettel(ctx context.Context, zettel domain.Zettel) bool // UpdateZettel updates an existing zettel. UpdateZettel(ctx context.Context, zettel domain.Zettel) error @@ -66,10 +64,18 @@ DeleteZettel(ctx context.Context, zid id.Zid) error // ReadStats populates st with place statistics ReadStats(st *Stats) } + +// ManagedPlace is the interface of managed places. +type ManagedPlace interface { + BasePlace + + // SelectMeta returns all zettel meta data that match the selection criteria. + SelectMeta(ctx context.Context, match search.MetaMatchFunc) ([]*meta.Meta, error) +} // Stats records statistics about the place. type Stats struct { // ReadOnly indicates that the places cannot be changed ReadOnly bool @@ -86,35 +92,23 @@ // Stop the started place. Now only the Start() function is allowed. Stop(ctx context.Context) error } -// ChangeReason gives an indication, why the ObserverFunc was called. -type ChangeReason int - -// Values for ChangeReason -const ( - _ ChangeReason = iota - OnReload // Place was reloaded - OnUpdate // A zettel was created or changed - OnDelete // A zettel was removed -) - -// ChangeInfo contains all the data about a changed zettel. -type ChangeInfo struct { - Reason ChangeReason - Zid id.Zid +// Place is a place to be used outside the place package and its descendants. +type Place interface { + BasePlace + + // SelectMeta returns a list of metadata that comply to the given selection criteria. + SelectMeta(ctx context.Context, s *search.Search) ([]*meta.Meta, error) } // Manager is a place-managing place. type Manager interface { Place StartStopper - - // RegisterObserver registers an observer that will be notified - // if one or all zettel are found to be changed. - RegisterObserver(func(ChangeInfo)) + change.Subject // NumPlaces returns the number of managed places. NumPlaces() int } @@ -179,23 +173,5 @@ // ErrInvalidID is returned if the zettel id is not appropriate for the place operation. type ErrInvalidID struct{ Zid id.Zid } func (err *ErrInvalidID) Error() string { return "invalid Zettel id: " + err.Zid.String() } - -// Filter specifies a mechanism for selecting zettel. -type Filter struct { - Expr FilterExpr - Negate bool - Select func(*meta.Meta) bool -} - -// FilterExpr is the encoding of a search filter. -type FilterExpr map[string][]string // map of keys to or-ed values - -// Sorter specifies ordering and limiting a sequnce of meta data. -type Sorter struct { - Order string // Name of meta key. None given: use "id" - Descending bool // Sort by order, but descending - Offset int // <= 0: no offset - Limit int // <= 0: no limit -} Index: place/progplace/indexer.go ================================================================== --- place/progplace/indexer.go +++ place/progplace/indexer.go @@ -41,7 +41,8 @@ fmt.Fprintf(&sb, "|Zettel| %v\n", stats.Store.Zettel) fmt.Fprintf(&sb, "|Last re-index| %v\n", stats.LastReload.Format("2006-01-02 15:04:05 -0700 MST")) fmt.Fprintf(&sb, "|Indexes since last re-index| %v\n", stats.IndexesSinceReload) fmt.Fprintf(&sb, "|Duration last index| %vms\n", stats.DurLastIndex.Milliseconds()) fmt.Fprintf(&sb, "|Zettel enrichments| %v\n", stats.Store.Updates) + fmt.Fprintf(&sb, "|Indexed words| %v\n", stats.Store.Words) return sb.String() } Index: place/progplace/progplace.go ================================================================== --- place/progplace/progplace.go +++ place/progplace/progplace.go @@ -20,16 +20,17 @@ "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/index" "zettelstore.de/z/place" "zettelstore.de/z/place/manager" + "zettelstore.de/z/search" ) func init() { manager.Register( " prog", - func(u *url.URL, cdata *manager.ConnectData) (place.Place, error) { + func(u *url.URL, cdata *manager.ConnectData) (place.ManagedPlace, error) { return getPlace(cdata.Filter), nil }) } type ( @@ -48,11 +49,11 @@ ) var myPlace *progPlace // Get returns the one program place. -func getPlace(mf index.MetaFilter) place.Place { +func getPlace(mf index.MetaFilter) place.ManagedPlace { if myPlace == nil { myPlace = &progPlace{ zettel: map[id.Zid]zettelGen{ id.Zid(1): {genVersionBuildM, genVersionBuildC}, id.Zid(2): {genVersionHostM, genVersionHostC}, @@ -91,12 +92,11 @@ func (pp *progPlace) CreateZettel( ctx context.Context, zettel domain.Zettel) (id.Zid, error) { return id.Invalid, place.ErrReadOnly } -func (pp *progPlace) GetZettel( - ctx context.Context, zid id.Zid) (domain.Zettel, error) { +func (pp *progPlace) GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error) { if gen, ok := pp.zettel[zid]; ok && gen.meta != nil { if m := gen.meta(zid); m != nil { updateMeta(m) if genContent := gen.content; genContent != nil { return domain.Zettel{ @@ -132,25 +132,23 @@ } } return result, nil } -func (pp *progPlace) SelectMeta( - ctx context.Context, f *place.Filter, s *place.Sorter) (res []*meta.Meta, err error) { - hasMatch := place.CreateFilterFunc(f) +func (pp *progPlace) SelectMeta(ctx context.Context, match search.MetaMatchFunc) (res []*meta.Meta, err error) { for zid, gen := range pp.zettel { if genMeta := gen.meta; genMeta != nil { if m := genMeta(zid); m != nil { updateMeta(m) pp.filter.Enrich(ctx, m) - if hasMatch(m) { + if match(m) { res = append(res, m) } } } } - return place.ApplySorter(res, s), nil + return res, nil } func (pp *progPlace) CanUpdateZettel(ctx context.Context, zettel domain.Zettel) bool { return false } @@ -184,12 +182,13 @@ st.ReadOnly = true st.Zettel = len(pp.zettel) } func updateMeta(m *meta.Meta) { + m.Set(meta.KeyNoIndex, meta.ValueTrue) m.Set(meta.KeySyntax, meta.ValueSyntaxZmk) m.Set(meta.KeyRole, meta.ValueRoleConfiguration) m.Set(meta.KeyReadOnly, meta.ValueTrue) if _, ok := m.Get(meta.KeyVisibility); !ok { m.Set(meta.KeyVisibility, meta.ValueVisibilityExpert) } } Index: place/progplace/runtime.go ================================================================== --- place/progplace/runtime.go +++ place/progplace/runtime.go @@ -11,13 +11,12 @@ // Package progplace provides zettel that inform the user about the internal Zettelstore state. package progplace import ( "fmt" - "runtime" + "runtime/metrics" "strings" - "time" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" ) @@ -24,35 +23,50 @@ func genRuntimeM(zid id.Zid) *meta.Meta { if myPlace.startConfig == nil { return nil } m := meta.New(zid) - m.Set(meta.KeyTitle, "Zettelstore Runtime Values") + m.Set(meta.KeyTitle, "Zettelstore Runtime Metrics") return m } func genRuntimeC(*meta.Meta) string { + var samples []metrics.Sample + all := metrics.All() + for _, d := range all { + if d.Kind == metrics.KindFloat64Histogram { + continue + } + samples = append(samples, metrics.Sample{Name: d.Name}) + } + metrics.Read(samples) + var sb strings.Builder sb.WriteString("|=Name|=Value>\n") - fmt.Fprintf(&sb, "|Number of CPUs|%v\n", runtime.NumCPU()) - fmt.Fprintf(&sb, "|Number of goroutines|%v\n", runtime.NumGoroutine()) - fmt.Fprintf(&sb, "|Number of Cgo calls|%v\n", runtime.NumCgoCall()) - var m runtime.MemStats - runtime.ReadMemStats(&m) - fmt.Fprintf(&sb, "|Memory from OS|%v\n", m.Sys) - fmt.Fprintf(&sb, "|Objects active|%v\n", m.Mallocs-m.Frees) - fmt.Fprintf(&sb, "|Heap alloc|%v\n", m.HeapAlloc) - fmt.Fprintf(&sb, "|Heap sys|%v\n", m.HeapSys) - fmt.Fprintf(&sb, "|Heap idle|%v\n", m.HeapIdle) - fmt.Fprintf(&sb, "|Heap in use|%v\n", m.HeapInuse) - fmt.Fprintf(&sb, "|Heap released|%v\n", m.HeapReleased) - fmt.Fprintf(&sb, "|Heap objects|%v\n", m.HeapObjects) - fmt.Fprintf(&sb, "|Stack in use|%v\n", m.StackInuse) - fmt.Fprintf(&sb, "|Stack sys|%v\n", m.StackSys) - fmt.Fprintf(&sb, "|Garbage collection metadata|%v\n", m.GCSys) - fmt.Fprintf(&sb, "|Last garbage collection|%v\n", time.Unix((int64)(m.LastGC/1000000000), 0)) - fmt.Fprintf(&sb, "|Garbage collection goal|%v\n", m.NextGC) - fmt.Fprintf(&sb, "|Garbage collections|%v\n", m.NumGC) - fmt.Fprintf(&sb, "|Forced garbage collections|%v\n", m.NumForcedGC) - fmt.Fprintf(&sb, "|Garbage collection fraction|%.3f%%\n", m.GCCPUFraction*100.0) + i := 0 + for _, d := range all { + if d.Kind == metrics.KindFloat64Histogram { + continue + } + descr := d.Description + if pos := strings.IndexByte(descr, '.'); pos > 0 { + descr = descr[:pos] + } + fmt.Fprintf(&sb, "|%s|", descr) + value := samples[i].Value + i++ + switch value.Kind() { + case metrics.KindUint64: + fmt.Fprintf(&sb, "%v", value.Uint64()) + case metrics.KindFloat64: + fmt.Fprintf(&sb, "%v", value.Float64()) + case metrics.KindFloat64Histogram: + sb.WriteString("???") + case metrics.KindBad: + sb.WriteString("BAD") + default: + fmt.Fprintf(&sb, "(unexpected metric kind: %v)", value.Kind()) + } + sb.WriteByte('\n') + } return sb.String() } DELETED place/sorter.go Index: place/sorter.go ================================================================== --- place/sorter.go +++ /dev/null @@ -1,145 +0,0 @@ -//----------------------------------------------------------------------------- -// Copyright (c) 2020-2021 Detlef Stern -// -// This file is part of zettelstore. -// -// Zettelstore is licensed under the latest version of the EUPL (European Union -// Public License). Please see file LICENSE.txt for your rights and obligations -// under this license. -//----------------------------------------------------------------------------- - -// Package place provides a generic interface to zettel places. -package place - -import ( - "math/rand" - "sort" - "strconv" - - "zettelstore.de/z/domain/meta" -) - -// RandomOrder is a pseudo metadata key that selects a random order. -const RandomOrder = "_random" - -// EnsureSorter makes sure that there is a sorter object. -func EnsureSorter(sorter *Sorter) *Sorter { - if sorter == nil { - sorter = new(Sorter) - } - return sorter -} - -// ApplySorter applies the given sorter to the slide of meta data. -func ApplySorter(metaList []*meta.Meta, s *Sorter) []*meta.Meta { - if len(metaList) == 0 { - return metaList - } - - if s == nil { - sort.Slice( - metaList, - func(i, j int) bool { - return metaList[i].Zid > metaList[j].Zid - }) - return metaList - } - - if s.Order == "" { - sort.Slice(metaList, createSortFunc(meta.KeyID, true, metaList)) - } else if s.Order == RandomOrder { - rand.Shuffle(len(metaList), func(i, j int) { - metaList[i], metaList[j] = metaList[j], metaList[i] - }) - } else { - sort.Slice(metaList, createSortFunc(s.Order, s.Descending, metaList)) - } - - if s.Offset > 0 { - if s.Offset > len(metaList) { - return nil - } - metaList = metaList[s.Offset:] - } - if s.Limit > 0 && s.Limit < len(metaList) { - metaList = metaList[:s.Limit] - } - return metaList -} - -type sortFunc func(i, j int) bool - -func createSortFunc(key string, descending bool, ml []*meta.Meta) sortFunc { - keyType := meta.Type(key) - if key == meta.KeyID || keyType == meta.TypeCredential { - if descending { - return func(i, j int) bool { return ml[i].Zid > ml[j].Zid } - } - return func(i, j int) bool { return ml[i].Zid < ml[j].Zid } - } - if keyType == meta.TypeBool { - return createSortBoolFunc(ml, key, descending) - } - if keyType == meta.TypeNumber { - return createSortNumberFunc(ml, key, descending) - } - return createSortStringFunc(ml, key, descending) -} - -func createSortBoolFunc(ml []*meta.Meta, key string, descending bool) sortFunc { - if descending { - return func(i, j int) bool { - left := ml[i].GetBool(key) - if left == ml[j].GetBool(key) { - return i > j - } - return left - } - } - return func(i, j int) bool { - right := ml[j].GetBool(key) - if ml[i].GetBool(key) == right { - return i < j - } - return right - } -} - -func createSortNumberFunc(ml []*meta.Meta, key string, descending bool) sortFunc { - if descending { - return func(i, j int) bool { - iVal, iOk := getNum(ml[i], key) - jVal, jOk := getNum(ml[j], key) - return (iOk && (!jOk || iVal > jVal)) || !jOk - } - } - return func(i, j int) bool { - iVal, iOk := getNum(ml[i], key) - jVal, jOk := getNum(ml[j], key) - return (iOk && (!jOk || iVal < jVal)) || !jOk - } -} - -func createSortStringFunc(ml []*meta.Meta, key string, descending bool) sortFunc { - if descending { - return func(i, j int) bool { - iVal, iOk := ml[i].Get(key) - jVal, jOk := ml[j].Get(key) - return (iOk && (!jOk || iVal > jVal)) || !jOk - } - } - return func(i, j int) bool { - iVal, iOk := ml[i].Get(key) - jVal, jOk := ml[j].Get(key) - return (iOk && (!jOk || iVal < jVal)) || !jOk - } -} - -func getNum(m *meta.Meta, key string) (int, bool) { - if s, ok := m.Get(key); ok { - if i, err := strconv.Atoi(s); err == nil { - return i, true - } - } - return 0, false -} Index: place/stock/stock.go ================================================================== --- place/stock/stock.go +++ place/stock/stock.go @@ -16,18 +16,16 @@ "sync" "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" - "zettelstore.de/z/place" + "zettelstore.de/z/place/change" ) // Place is a place that is used by a stock. type Place interface { - // RegisterObserver registers an observer that will be notified - // if all or one zettel are found to be changed. - RegisterObserver(ob func(place.ChangeInfo)) + change.Subject // GetZettel retrieves a specific zettel. GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error) } @@ -38,11 +36,10 @@ GetMeta(zid id.Zid) *meta.Meta } // NewStock creates a new stock that operates on the given place. func NewStock(place Place) Stock { - //RegisterChangeObserver(func(domain.Zid)) stock := &defaultStock{ place: place, subs: make(map[id.Zid]domain.Zettel), } place.RegisterObserver(stock.observe) @@ -54,12 +51,12 @@ subs map[id.Zid]domain.Zettel mxSubs sync.RWMutex } // observe tracks all changes the place signals. -func (s *defaultStock) observe(ci place.ChangeInfo) { - if ci.Reason == place.OnReload { +func (s *defaultStock) observe(ci change.Info) { + if ci.Reason == change.OnReload { go func() { s.mxSubs.Lock() defer s.mxSubs.Unlock() for zid := range s.subs { s.update(zid) ADDED search/filter.go Index: search/filter.go ================================================================== --- /dev/null +++ search/filter.go @@ -0,0 +1,307 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2020-2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package search provides a zettel search. +package search + +import ( + "strings" + + "zettelstore.de/z/domain/meta" +) + +type matchFunc func(value string) bool + +func matchNever(value string) bool { return false } +func matchAlways(value string) bool { return true } + +type matchSpec struct { + key string + match matchFunc +} + +// compileFilter calculates a filter func based on the given filter. +func compileFilter(tags expTagValues) MetaMatchFunc { + specs, nomatch := createFilterSpecs(tags) + if len(specs) == 0 && len(nomatch) == 0 { + return nil + } + return makeSearchMetaFilterFunc(specs, nomatch) +} + +func createFilterSpecs(tags map[string][]expValue) ([]matchSpec, []string) { + specs := make([]matchSpec, 0, len(tags)) + var nomatch []string + for key, values := range tags { + if !meta.KeyIsValid(key) { + continue + } + if empty, negates := hasEmptyValues(values); empty { + if negates == 0 { + specs = append(specs, matchSpec{key, matchAlways}) + continue + } + if len(values) < negates { + specs = append(specs, matchSpec{key, matchNever}) + continue + } + nomatch = append(nomatch, key) + continue + } + match := createMatchFunc(key, values) + if match != nil { + specs = append(specs, matchSpec{key, match}) + } + } + return specs, nomatch +} + +func hasEmptyValues(values []expValue) (bool, int) { + var negates int + for _, v := range values { + if v.value != "" { + continue + } + if !v.negate { + return true, 0 + } + negates++ + } + return negates > 0, negates +} + +func createMatchFunc(key string, values []expValue) matchFunc { + switch meta.Type(key) { + case meta.TypeBool: + return createMatchBoolFunc(values) + case meta.TypeCredential: + return matchNever + case meta.TypeID, meta.TypeTimestamp: // ID and timestamp use the same layout + return createMatchIDFunc(values) + case meta.TypeIDSet: + return createMatchIDSetFunc(values) + case meta.TypeTagSet: + return createMatchTagSetFunc(values) + case meta.TypeWord: + return createMatchWordFunc(values) + case meta.TypeWordSet: + return createMatchWordSetFunc(values) + } + return createMatchStringFunc(values) +} + +func createMatchBoolFunc(values []expValue) matchFunc { + preValues := make([]bool, 0, len(values)) + for _, v := range values { + boolValue := meta.BoolValue(v.value) + if v.negate { + boolValue = !boolValue + } + preValues = append(preValues, boolValue) + } + return func(value string) bool { + bValue := meta.BoolValue(value) + for _, v := range preValues { + if bValue != v { + return false + } + } + return true + } +} + +func createMatchIDFunc(values []expValue) matchFunc { + return func(value string) bool { + for _, v := range values { + if strings.HasPrefix(value, v.value) == v.negate { + return false + } + } + return true + } +} + +func createMatchIDSetFunc(values []expValue) matchFunc { + idValues := preprocessSet(sliceToLower(values)) + return func(value string) bool { + ids := meta.ListFromValue(value) + for _, neededIDs := range idValues { + for _, neededID := range neededIDs { + if matchAllID(ids, neededID.value) == neededID.negate { + return false + } + } + } + return true + } +} + +func matchAllID(zettelIDs []string, neededID string) bool { + for _, zt := range zettelIDs { + if strings.HasPrefix(zt, neededID) { + return true + } + } + return false +} + +func createMatchTagSetFunc(values []expValue) matchFunc { + tagValues := processTagSet(preprocessSet(values)) + return func(value string) bool { + tags := meta.ListFromValue(value) + // Remove leading '#' from each tag + for i, tag := range tags { + tags[i] = meta.CleanTag(tag) + } + for _, neededTags := range tagValues { + for _, neededTag := range neededTags { + if matchAllTag(tags, neededTag.value, neededTag.equal) == neededTag.negate { + return false + } + } + } + return true + } +} + +type tagQueryValue struct { + value string + negate bool + equal bool // not equal == prefix +} + +func processTagSet(valueSet [][]expValue) [][]tagQueryValue { + result := make([][]tagQueryValue, len(valueSet)) + for i, values := range valueSet { + tags := make([]tagQueryValue, len(values)) + for j, val := range values { + if tval := val.value; tval != "" && tval[0] == '#' { + tval = meta.CleanTag(tval) + tags[j] = tagQueryValue{value: tval, negate: val.negate, equal: true} + } else { + tags[j] = tagQueryValue{value: tval, negate: val.negate, equal: false} + } + } + result[i] = tags + } + return result +} + +func matchAllTag(zettelTags []string, neededTag string, equal bool) bool { + if equal { + for _, zt := range zettelTags { + if zt == neededTag { + return true + } + } + } else { + for _, zt := range zettelTags { + if strings.HasPrefix(zt, neededTag) { + return true + } + } + } + return false +} + +func createMatchWordFunc(values []expValue) matchFunc { + values = sliceToLower(values) + return func(value string) bool { + value = strings.ToLower(value) + for _, v := range values { + if (value == v.value) == v.negate { + return false + } + } + return true + } +} + +func createMatchWordSetFunc(values []expValue) matchFunc { + wordValues := preprocessSet(sliceToLower(values)) + return func(value string) bool { + words := meta.ListFromValue(value) + for _, neededWords := range wordValues { + for _, neededWord := range neededWords { + if matchAllWord(words, neededWord.value) == neededWord.negate { + return false + } + } + } + return true + } +} + +func createMatchStringFunc(values []expValue) matchFunc { + values = sliceToLower(values) + return func(value string) bool { + value = strings.ToLower(value) + for _, v := range values { + if strings.Contains(value, v.value) == v.negate { + return false + } + } + return true + } +} + +func makeSearchMetaFilterFunc(specs []matchSpec, nomatch []string) MetaMatchFunc { + return func(m *meta.Meta) bool { + for _, s := range specs { + if value, ok := m.Get(s.key); !ok || !s.match(value) { + return false + } + } + for _, key := range nomatch { + if _, ok := m.Get(key); ok { + return false + } + } + return true + } +} + +func sliceToLower(sl []expValue) []expValue { + result := make([]expValue, 0, len(sl)) + for _, s := range sl { + result = append(result, expValue{ + value: strings.ToLower(s.value), + negate: s.negate, + }) + } + return result +} + +func preprocessSet(set []expValue) [][]expValue { + result := make([][]expValue, 0, len(set)) + for _, elem := range set { + splitElems := strings.Split(elem.value, ",") + valueElems := make([]expValue, 0, len(splitElems)) + for _, se := range splitElems { + e := strings.TrimSpace(se) + if len(e) > 0 { + valueElems = append(valueElems, expValue{value: e, negate: elem.negate}) + } + } + if len(valueElems) > 0 { + result = append(result, valueElems) + } + } + return result +} + +func matchAllWord(zettelWords []string, neededWord string) bool { + for _, zw := range zettelWords { + if zw == neededWord { + return true + } + } + return false +} ADDED search/print.go Index: search/print.go ================================================================== --- /dev/null +++ search/print.go @@ -0,0 +1,106 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2020-2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package search provides a zettel search. +package search + +import ( + "io" + "sort" + "strconv" + + "zettelstore.de/z/domain/meta" +) + +// Print the filter to a writer. +func (s *Search) Print(w io.Writer) { + if s.negate { + io.WriteString(w, "NOT (") + } + space := false + if len(s.search) > 0 { + io.WriteString(w, "ANY") + printFilterExprValues(w, s.search) + space = true + } + names := make([]string, 0, len(s.tags)) + for name := range s.tags { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if space { + io.WriteString(w, " AND ") + } + io.WriteString(w, name) + printFilterExprValues(w, s.tags[name]) + space = true + } + if s.negate { + io.WriteString(w, ")") + space = true + } + + if ord := s.order; len(ord) > 0 { + switch ord { + case meta.KeyID: + // Ignore + case RandomOrder: + space = printSpace(w, space) + io.WriteString(w, "RANDOM") + default: + space = printSpace(w, space) + io.WriteString(w, "SORT ") + io.WriteString(w, ord) + if s.descending { + io.WriteString(w, " DESC") + } + } + } + if off := s.offset; off > 0 { + space = printSpace(w, space) + io.WriteString(w, "OFFSET ") + io.WriteString(w, strconv.Itoa(off)) + } + if lim := s.limit; lim > 0 { + _ = printSpace(w, space) + io.WriteString(w, "LIMIT ") + io.WriteString(w, strconv.Itoa(lim)) + } +} + +func printFilterExprValues(w io.Writer, values []expValue) { + if len(values) == 0 { + io.WriteString(w, " MATCH ANY") + return + } + + for j, val := range values { + if j > 0 { + io.WriteString(w, " AND") + } + if val.negate { + io.WriteString(w, " NOT") + } + io.WriteString(w, " MATCH ") + if val.value == "" { + io.WriteString(w, "ANY") + } else { + io.WriteString(w, val.value) + } + } +} + +func printSpace(w io.Writer, space bool) bool { + if space { + io.WriteString(w, " ") + } + return true +} ADDED search/search.go Index: search/search.go ================================================================== --- /dev/null +++ search/search.go @@ -0,0 +1,278 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2020-2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package search provides a zettel search. +package search + +import ( + "math/rand" + "sort" + "sync" + + "zettelstore.de/z/domain/meta" + "zettelstore.de/z/index" +) + +// MetaMatchFunc is a function determine whethe some metadata should be filtered or not. +type MetaMatchFunc func(*meta.Meta) bool + +// Search specifies a mechanism for selecting zettel. +type Search struct { + mx sync.RWMutex // Protects other attributes + + // Fields to be used for filtering + preMatch MetaMatchFunc // Match that must be true + tags expTagValues // Expected values for a tag + search []expValue // Search string + negate bool // Negate the result of the whole filtering process + + // Fields to be used for sorting + order string // Name of meta key. None given: use "id" + descending bool // Sort by order, but descending + offset int // <= 0: no offset + limit int // <= 0: no limit +} + +type expTagValues map[string][]expValue + +// RandomOrder is a pseudo metadata key that selects a random order. +const RandomOrder = "_random" + +type expValue struct { + value string + negate bool +} + +// AddExpr adds a match expression to the filter. +func (s *Search) AddExpr(key, val string, negate bool) *Search { + if s == nil { + s = new(Search) + } + s.mx.Lock() + defer s.mx.Unlock() + if key == "" { + s.search = append(s.search, expValue{value: val, negate: negate}) + } else if s.tags == nil { + s.tags = expTagValues{key: {{value: val, negate: negate}}} + } else { + s.tags[key] = append(s.tags[key], expValue{value: val, negate: negate}) + } + return s +} + +// SetNegate changes the filter to reverse its selection. +func (s *Search) SetNegate() *Search { + if s == nil { + s = new(Search) + } + s.mx.Lock() + defer s.mx.Unlock() + s.negate = true + return s +} + +// AddPreMatch adds the pre-filter selection predicate. +func (s *Search) AddPreMatch(preMatch MetaMatchFunc) *Search { + if s == nil { + s = new(Search) + } + s.mx.Lock() + defer s.mx.Unlock() + if pre := s.preMatch; pre == nil { + s.preMatch = preMatch + } else { + s.preMatch = func(m *meta.Meta) bool { + return preMatch(m) && pre(m) + } + } + return s +} + +// AddOrder adds the given order to the search object. +func (s *Search) AddOrder(key string, descending bool) *Search { + if s == nil { + s = new(Search) + } + s.mx.Lock() + defer s.mx.Unlock() + if s.order != "" { + panic("order field already set: " + s.order) + } + s.order = key + s.descending = descending + return s +} + +// SetOffset sets the given offset of the search object. +func (s *Search) SetOffset(offset int) *Search { + if s == nil { + s = new(Search) + } + s.mx.Lock() + defer s.mx.Unlock() + if offset < 0 { + offset = 0 + } + s.offset = offset + return s +} + +// GetOffset returns the current offset value. +func (s *Search) GetOffset() int { + if s == nil { + return 0 + } + s.mx.RLock() + defer s.mx.RUnlock() + return s.offset +} + +// SetLimit sets the given limit of the search object. +func (s *Search) SetLimit(limit int) *Search { + if s == nil { + s = new(Search) + } + s.mx.Lock() + defer s.mx.Unlock() + if limit < 0 { + limit = 0 + } + s.limit = limit + return s +} + +// GetLimit returns the current offset value. +func (s *Search) GetLimit() int { + if s == nil { + return 0 + } + s.mx.RLock() + defer s.mx.RUnlock() + return s.limit +} + +// HasComputedMetaKey returns true, if the filter references a metadata key which +// a computed value. +func (s *Search) HasComputedMetaKey() bool { + if s == nil { + return false + } + s.mx.RLock() + defer s.mx.RUnlock() + for key := range s.tags { + if meta.IsComputed(key) { + return true + } + } + if order := s.order; order != "" && meta.IsComputed(order) { + return true + } + return false +} + +// CompileMatch returns a function to match meta data based on filter specification. +func (s *Search) CompileMatch(selector index.Selector) MetaMatchFunc { + if s == nil { + return filterNone + } + s.mx.Lock() + defer s.mx.Unlock() + + compMeta := compileFilter(s.tags) + //compSearch := createSearchAllFunc(s.search) + compSearch := compileSearch(selector, s.search) + + if preMatch := s.preMatch; preMatch != nil { + return compilePreMatch(preMatch, compMeta, compSearch, s.negate) + } + return compileNoPreMatch(compMeta, compSearch, s.negate) +} + +func filterNone(m *meta.Meta) bool { return true } + +func compilePreMatch(preMatch, compMeta, compSearch MetaMatchFunc, negate bool) MetaMatchFunc { + if compMeta == nil { + if compSearch == nil { + return preMatch + } + if negate { + return func(m *meta.Meta) bool { return preMatch(m) && !compSearch(m) } + } + return func(m *meta.Meta) bool { return preMatch(m) && compSearch(m) } + } + if compSearch == nil { + if negate { + return func(m *meta.Meta) bool { return preMatch(m) && !compMeta(m) } + } + return func(m *meta.Meta) bool { return preMatch(m) && compMeta(m) } + } + if negate { + return func(m *meta.Meta) bool { return preMatch(m) && (!compMeta(m) || !compSearch(m)) } + } + return func(m *meta.Meta) bool { return preMatch(m) && compMeta(m) && compSearch(m) } +} + +func compileNoPreMatch(compMeta, compSearch MetaMatchFunc, negate bool) MetaMatchFunc { + if compMeta == nil { + if compSearch == nil { + if negate { + return func(m *meta.Meta) bool { return false } + } + return filterNone + } + if negate { + return func(m *meta.Meta) bool { return !compSearch(m) } + } + return compSearch + } + if compSearch == nil { + if negate { + return func(m *meta.Meta) bool { return !compMeta(m) } + } + return compMeta + } + if negate { + return func(m *meta.Meta) bool { return !compMeta(m) || !compSearch(m) } + } + return func(m *meta.Meta) bool { return compMeta(m) && compSearch(m) } +} + +// Sort applies the sorter to the slice of meta data. +func (s *Search) Sort(metaList []*meta.Meta) []*meta.Meta { + if len(metaList) == 0 { + return metaList + } + + if s == nil { + sort.Slice(metaList, func(i, j int) bool { return metaList[i].Zid > metaList[j].Zid }) + return metaList + } + + if s.order == "" { + sort.Slice(metaList, createSortFunc(meta.KeyID, true, metaList)) + } else if s.order == RandomOrder { + rand.Shuffle(len(metaList), func(i, j int) { + metaList[i], metaList[j] = metaList[j], metaList[i] + }) + } else { + sort.Slice(metaList, createSortFunc(s.order, s.descending, metaList)) + } + + if s.offset > 0 { + if s.offset > len(metaList) { + return nil + } + metaList = metaList[s.offset:] + } + if s.limit > 0 && s.limit < len(metaList) { + metaList = metaList[:s.limit] + } + return metaList +} ADDED search/selector.go Index: search/selector.go ================================================================== --- /dev/null +++ search/selector.go @@ -0,0 +1,95 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package search provides a zettel search. +package search + +// This file is about "compiling" a search expression into a function. + +import ( + "zettelstore.de/z/domain/id" + "zettelstore.de/z/domain/meta" + "zettelstore.de/z/index" + "zettelstore.de/z/strfun" +) + +func compileSearch(selector index.Selector, search []expValue) MetaMatchFunc { + poss, negs := normalizeSearchValues(search) + if len(poss) == 0 { + if len(negs) == 0 { + return nil + } + return makeNegOnlySearch(selector, negs) + } + if len(negs) == 0 { + return makePosOnlySearch(selector, poss) + } + return makePosNegSearch(selector, poss, negs) +} + +func normalizeSearchValues(search []expValue) (positives, negatives []string) { + posSet := make(map[string]bool) + negSet := make(map[string]bool) + for _, val := range search { + for _, word := range strfun.NormalizeWords(val.value) { + if val.negate { + if _, ok := negSet[word]; !ok { + negSet[word] = true + negatives = append(negatives, word) + } + } else { + if _, ok := posSet[word]; !ok { + posSet[word] = true + positives = append(positives, word) + } + } + } + } + return positives, negatives +} + +func makePosOnlySearch(selector index.Selector, poss []string) MetaMatchFunc { + return func(m *meta.Meta) bool { + ids := retrieveZids(selector, poss) + _, ok := ids[m.Zid] + return ok + } +} + +func makeNegOnlySearch(selector index.Selector, negs []string) MetaMatchFunc { + return func(m *meta.Meta) bool { + ids := retrieveZids(selector, negs) + _, ok := ids[m.Zid] + return !ok + } +} + +func makePosNegSearch(selector index.Selector, poss, negs []string) MetaMatchFunc { + return func(m *meta.Meta) bool { + idsPos := retrieveZids(selector, poss) + _, okPos := idsPos[m.Zid] + idsNeg := retrieveZids(selector, negs) + _, okNeg := idsNeg[m.Zid] + return okPos && !okNeg + } +} + +func retrieveZids(selector index.Selector, words []string) id.Set { + var result id.Set + for i, word := range words { + ids := selector.SelectContains(word) + if i == 0 { + result = ids + continue + } + result = result.Intersect(ids) + } + return result +} ADDED search/sorter.go Index: search/sorter.go ================================================================== --- /dev/null +++ search/sorter.go @@ -0,0 +1,95 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2020-2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package search provides a zettel search. +package search + +import ( + "strconv" + + "zettelstore.de/z/domain/meta" +) + +type sortFunc func(i, j int) bool + +func createSortFunc(key string, descending bool, ml []*meta.Meta) sortFunc { + keyType := meta.Type(key) + if key == meta.KeyID || keyType == meta.TypeCredential { + if descending { + return func(i, j int) bool { return ml[i].Zid > ml[j].Zid } + } + return func(i, j int) bool { return ml[i].Zid < ml[j].Zid } + } + if keyType == meta.TypeBool { + return createSortBoolFunc(ml, key, descending) + } + if keyType == meta.TypeNumber { + return createSortNumberFunc(ml, key, descending) + } + return createSortStringFunc(ml, key, descending) +} + +func createSortBoolFunc(ml []*meta.Meta, key string, descending bool) sortFunc { + if descending { + return func(i, j int) bool { + left := ml[i].GetBool(key) + if left == ml[j].GetBool(key) { + return i > j + } + return left + } + } + return func(i, j int) bool { + right := ml[j].GetBool(key) + if ml[i].GetBool(key) == right { + return i < j + } + return right + } +} + +func createSortNumberFunc(ml []*meta.Meta, key string, descending bool) sortFunc { + if descending { + return func(i, j int) bool { + iVal, iOk := getNum(ml[i], key) + jVal, jOk := getNum(ml[j], key) + return (iOk && (!jOk || iVal > jVal)) || !jOk + } + } + return func(i, j int) bool { + iVal, iOk := getNum(ml[i], key) + jVal, jOk := getNum(ml[j], key) + return (iOk && (!jOk || iVal < jVal)) || !jOk + } +} + +func createSortStringFunc(ml []*meta.Meta, key string, descending bool) sortFunc { + if descending { + return func(i, j int) bool { + iVal, iOk := ml[i].Get(key) + jVal, jOk := ml[j].Get(key) + return (iOk && (!jOk || iVal > jVal)) || !jOk + } + } + return func(i, j int) bool { + iVal, iOk := ml[i].Get(key) + jVal, jOk := ml[j].Get(key) + return (iOk && (!jOk || iVal < jVal)) || !jOk + } +} + +func getNum(m *meta.Meta, key string) (int, bool) { + if s, ok := m.Get(key); ok { + if i, err := strconv.Atoi(s); err == nil { + return i, true + } + } + return 0, false +} ADDED staticcheck.conf Index: staticcheck.conf ================================================================== --- /dev/null +++ staticcheck.conf @@ -0,0 +1,2 @@ +checks = ["all"] +http_status_code_whitelist = [] Index: strfun/slugify.go ================================================================== --- strfun/slugify.go +++ strfun/slugify.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -10,11 +10,10 @@ // Package strfun provides some string functions. package strfun import ( - "strings" "unicode" "golang.org/x/text/unicode/norm" ) @@ -30,11 +29,10 @@ } ) // Slugify returns a string that can be used as part of an URL func Slugify(s string) string { - s = strings.TrimSpace(s) result := make([]rune, 0, len(s)) addDash := false for _, r := range norm.NFKD.String(s) { if unicode.IsOneOf(useUnicode, r) { result = append(result, unicode.ToLower(r)) @@ -47,5 +45,23 @@ if i := len(result) - 1; i >= 0 && result[i] == '-' { result = result[:i] } return string(result) } + +// NormalizeWords produces a word list that is normalized for better searching. +func NormalizeWords(s string) []string { + result := make([]string, 0, 1) + word := make([]rune, 0, len(s)) + for _, r := range norm.NFKD.String(s) { + if unicode.IsOneOf(useUnicode, r) { + word = append(word, unicode.ToLower(r)) + } else if !unicode.IsOneOf(ignoreUnicode, r) && len(word) > 0 { + result = append(result, string(word)) + word = word[:0] + } + } + if len(word) > 0 { + result = append(result, string(word)) + } + return result +} Index: strfun/slugify_test.go ================================================================== --- strfun/slugify_test.go +++ strfun/slugify_test.go @@ -1,7 +1,7 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations @@ -15,22 +15,57 @@ "testing" "zettelstore.de/z/strfun" ) -var tests = []struct{ in, exp string }{ - {"simple test", "simple-test"}, - {"I'm a go developer", "i-m-a-go-developer"}, - {"-!->simple test<-!-", "simple-test"}, - {"äöüÄÖÜß", "aouaouß"}, - {"\"aèf", "aef"}, - {"a#b", "a-b"}, - {"*", ""}, -} - func TestSlugify(t *testing.T) { + tests := []struct{ in, exp string }{ + {"simple test", "simple-test"}, + {"I'm a go developer", "i-m-a-go-developer"}, + {"-!->simple test<-!-", "simple-test"}, + {"äöüÄÖÜß", "aouaouß"}, + {"\"aèf", "aef"}, + {"a#b", "a-b"}, + {"*", ""}, + } for _, test := range tests { if got := strfun.Slugify(test.in); got != test.exp { t.Errorf("%q: %q != %q", test.in, got, test.exp) } } } + +func eqStringSlide(got, exp []string) bool { + if len(got) != len(exp) { + return false + } + for i, g := range got { + if g != exp[i] { + return false + } + } + return true +} + +func TestNormalizeWord(t *testing.T) { + tests := []struct { + in string + exp []string + }{ + {"simple test", []string{"simple", "test"}}, + {"I'm a go developer", []string{"i", "m", "a", "go", "developer"}}, + {"-!->simple test<-!-", []string{"simple", "test"}}, + {"äöüÄÖÜß", []string{"aouaouß"}}, + {"\"aèf", []string{"aef"}}, + {"a#b", []string{"a", "b"}}, + {"*", []string{}}, + {"123", []string{"123"}}, + {"1²3", []string{"123"}}, + {"Period.", []string{"period"}}, + {" WORD NUMBER ", []string{"word", "number"}}, + } + for _, test := range tests { + if got := strfun.NormalizeWords(test.in); !eqStringSlide(got, test.exp) { + t.Errorf("%q: %q != %q", test.in, got, test.exp) + } + } +} Index: template/mustache.go ================================================================== --- template/mustache.go +++ template/mustache.go @@ -16,10 +16,11 @@ // above MIT license. // The license text is included in the same directory where this file is // located. See file LICENSE. //----------------------------------------------------------------------------- +// Package template implements the Mustache templating language. package template import ( "fmt" "io" @@ -261,39 +262,43 @@ eow = i break } } - // Skip all whitespaces apeared after these types of tags until end of line if - // the line only contains a tag and whitespaces. - standalone := true - if mayStandalone { - if _, ok := skipWhitespaceTagTypes[tag[0]]; !ok { - standalone = false - } else { - if eow == len(tmpl.data) { - standalone = true - tmpl.p = eow - } else if eow < len(tmpl.data) && tmpl.data[eow] == '\n' { - standalone = true - tmpl.p = eow + 1 - tmpl.curline++ - } else if eow+1 < len(tmpl.data) && tmpl.data[eow] == '\r' && tmpl.data[eow+1] == '\n' { - standalone = true - tmpl.p = eow + 2 - tmpl.curline++ - } else { - standalone = false - } - } - } + standalone := tmpl.skipWhitespaceTag(tag, eow, mayStandalone) return &tagReadingResult{ tag: tag, standalone: standalone, }, nil } + +func (tmpl *Template) skipWhitespaceTag(tag string, eow int, mayStandalone bool) bool { + if !mayStandalone { + return true + } + // Skip all whitespaces apeared after these types of tags until end of line if + // the line only contains a tag and whitespaces. + if _, ok := skipWhitespaceTagTypes[tag[0]]; !ok { + return false + } + if eow == len(tmpl.data) { + tmpl.p = eow + return true + } + if eow < len(tmpl.data) && tmpl.data[eow] == '\n' { + tmpl.p = eow + 1 + tmpl.curline++ + return true + } + if eow+1 < len(tmpl.data) && tmpl.data[eow] == '\r' && tmpl.data[eow+1] == '\n' { + tmpl.p = eow + 2 + tmpl.curline++ + return true + } + return false +} func (tmpl *Template) parsePartial(name, indent string) (*partialNode, error) { return &partialNode{ name: name, indent: indent, @@ -457,54 +462,58 @@ return v, err } return lookup([]reflect.Value{v}, name[pos+1:], errMissing) } -Outer: for i := len(stack) - 1; i >= 0; i-- { - v := stack[i] - for v.IsValid() { - typ := v.Type() - if n := v.Type().NumMethod(); n > 0 { - for i := 0; i < n; i++ { - m := typ.Method(i) - mtyp := m.Type - if m.Name == name && mtyp.NumIn() == 1 { - return v.Method(i).Call(nil)[0], nil - } - } - } - if name == "." { - return v, nil - } - switch av := v; av.Kind() { - case reflect.Ptr: - v = av.Elem() - case reflect.Interface: - v = av.Elem() - case reflect.Struct: - ret := av.FieldByName(name) - if ret.IsValid() { - return ret, nil - } - continue Outer - case reflect.Map: - ret := av.MapIndex(reflect.ValueOf(name)) - if ret.IsValid() { - return ret, nil - } - continue Outer - default: - continue Outer - } + if val, ok := lookupValue(stack[i], name); ok { + return val, nil } } if errMissing { return reflect.Value{}, fmt.Errorf("missing variable %q", name) } return reflect.Value{}, nil } + +func lookupValue(v reflect.Value, name string) (reflect.Value, bool) { + for v.IsValid() { + typ := v.Type() + if n := v.Type().NumMethod(); n > 0 { + for i := 0; i < n; i++ { + m := typ.Method(i) + mtyp := m.Type + if m.Name == name && mtyp.NumIn() == 1 { + return v.Method(i).Call(nil)[0], true + } + } + } + if name == "." { + return v, true + } + switch av := v; av.Kind() { + case reflect.Ptr: + v = av.Elem() + case reflect.Interface: + v = av.Elem() + case reflect.Struct: + return sanitizeValue(av.FieldByName(name)) + case reflect.Map: + return sanitizeValue(av.MapIndex(reflect.ValueOf(name))) + default: + return reflect.Value{}, false + } + } + return reflect.Value{}, false +} + +func sanitizeValue(v reflect.Value) (reflect.Value, bool) { + if v.IsValid() { + return v, true + } + return reflect.Value{}, false +} func isEmpty(v reflect.Value) bool { if !v.IsValid() || v.Interface() == nil { return true } @@ -542,43 +551,43 @@ value, err := lookup(stack, section.name, false) if err != nil { return err } - // if the value is nil, check if it's an inverted section - isEmpty := isEmpty(value) - if isEmpty && !section.inverted || !isEmpty && section.inverted { + // if the value is empty, check if it's an inverted section + if isEmpty(value) != section.inverted { return nil } if !section.inverted { switch val := indirect(value); val.Kind() { case reflect.Slice, reflect.Array: valLen := val.Len() - enumeration := make([]reflect.Value, 0, valLen) + enumeration := make([]reflect.Value, valLen) for i := 0; i < valLen; i++ { - enumeration = append(enumeration, val.Index(i)) + enumeration[i] = val.Index(i) } topStack := len(stack) stack = append(stack, enumeration[0]) - defer func() { stack = stack[:topStack-1] }() for _, elem := range enumeration { stack[topStack] = elem - for _, n := range section.nodes { - if err := tmpl.renderNode(w, n, stack); err != nil { - return err - } + if err := tmpl.renderNodes(w, section.nodes, stack); err != nil { + return err } } return nil case reflect.Map, reflect.Struct: - stack = append(stack, value) - defer func() { stack = stack[:len(stack)-2] }() + return tmpl.renderNodes(w, section.nodes, append(stack, value)) } + return tmpl.renderNodes(w, section.nodes, stack) } - for _, n := range section.nodes { + return tmpl.renderNodes(w, section.nodes, stack) +} + +func (tmpl *Template) renderNodes(w io.Writer, nodes []node, stack []reflect.Value) error { + for _, n := range nodes { if err := tmpl.renderNode(w, n, stack); err != nil { return err } } return nil Index: template/mustache_test.go ================================================================== --- template/mustache_test.go +++ template/mustache_test.go @@ -307,25 +307,10 @@ } } } } -type LayoutTest struct { - layout string - tmpl string - context interface{} - expected string -} - -var layoutTests = []LayoutTest{ - {`Header {{content}} Footer`, `Hello World`, nil, `Header Hello World Footer`}, - {`Header {{content}} Footer`, `Hello {{s}}`, map[string]string{"s": "World"}, `Header Hello World Footer`}, - {`Header {{content}} Footer`, `Hello {{content}}`, map[string]string{"content": "World"}, `Header Hello World Footer`}, - {`Header {{extra}} {{content}} Footer`, `Hello {{content}}`, map[string]string{"content": "World", "extra": "extra"}, `Header extra Hello World Footer`}, - {`Header {{content}} {{content}} Footer`, `Hello {{content}}`, map[string]string{"content": "World"}, `Header Hello World Hello World Footer`}, -} - type Person struct { FirstName string LastName string } Index: template/spec_test.go ================================================================== --- template/spec_test.go +++ template/spec_test.go @@ -20,11 +20,10 @@ package template_test import ( "encoding/json" - "io/ioutil" "os" "path/filepath" "sort" "testing" @@ -204,11 +203,11 @@ continue } if enabled == nil { continue } - b, err := ioutil.ReadFile(path) + b, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var suite specTestSuite err = json.Unmarshal(b, &suite) Index: tests/markdown_test.go ================================================================== --- tests/markdown_test.go +++ tests/markdown_test.go @@ -12,11 +12,11 @@ package tests import ( "encoding/json" "fmt" - "io/ioutil" + "os" "regexp" "strings" "testing" "zettelstore.de/z/ast" @@ -68,11 +68,11 @@ var reHeadingID = regexp.MustCompile(` id="[^"]*"`) func TestEncoderAvailability(t *testing.T) { encoderMissing := false for _, format := range formats { - enc := encoder.Create(format) + enc := encoder.Create(format, nil) if enc == nil { t.Errorf("No encoder for %q found", format) encoderMissing = true } } @@ -80,11 +80,11 @@ panic("At least one encoder is missing. See test log") } } func TestMarkdownSpec(t *testing.T) { - content, err := ioutil.ReadFile("../testdata/markdown/spec.json") + content, err := os.ReadFile("../testdata/markdown/spec.json") if err != nil { panic(err) } var testcases []markdownTestCase if err = json.Unmarshal(content, &testcases); err != nil { @@ -107,18 +107,18 @@ func testAllEncodings(t *testing.T, tc markdownTestCase, ast ast.BlockSlice) { var sb strings.Builder testID := tc.Example*100 + 1 for _, format := range formats { t.Run(fmt.Sprintf("Encode %v %v", format, testID), func(st *testing.T) { - encoder.Create(format).WriteBlocks(&sb, ast) + encoder.Create(format, nil).WriteBlocks(&sb, ast) sb.Reset() }) } } func testHTMLEncoding(t *testing.T, tc markdownTestCase, ast ast.BlockSlice) { - htmlEncoder := encoder.Create("html", &encoder.BoolOption{Key: "xhtml", Value: true}) + htmlEncoder := encoder.Create("html", &encoder.Environment{Xhtml: true}) var sb strings.Builder testID := tc.Example*100 + 1 t.Run(fmt.Sprintf("Encode md html %v", testID), func(st *testing.T) { htmlEncoder.WriteBlocks(&sb, ast) gotHTML := sb.String() @@ -139,11 +139,11 @@ } }) } func testZmkEncoding(t *testing.T, tc markdownTestCase, ast ast.BlockSlice) { - zmkEncoder := encoder.Create("zmk") + zmkEncoder := encoder.Create("zmk", nil) var sb strings.Builder testID := tc.Example*100 + 1 t.Run(fmt.Sprintf("Encode zmk %14d", testID), func(st *testing.T) { zmkEncoder.WriteBlocks(&sb, ast) gotFirst := sb.String() Index: tests/regression_test.go ================================================================== --- tests/regression_test.go +++ tests/regression_test.go @@ -12,11 +12,11 @@ package tests import ( "context" "fmt" - "io/ioutil" + "io" "os" "path/filepath" "strings" "testing" @@ -24,36 +24,36 @@ "zettelstore.de/z/domain" "zettelstore.de/z/domain/meta" "zettelstore.de/z/encoder" "zettelstore.de/z/parser" "zettelstore.de/z/place" + "zettelstore.de/z/place/manager" _ "zettelstore.de/z/encoder/htmlenc" _ "zettelstore.de/z/encoder/jsonenc" _ "zettelstore.de/z/encoder/nativeenc" _ "zettelstore.de/z/encoder/textenc" _ "zettelstore.de/z/encoder/zmkenc" _ "zettelstore.de/z/parser/blob" _ "zettelstore.de/z/parser/zettelmark" _ "zettelstore.de/z/place/dirplace" - "zettelstore.de/z/place/manager" ) var formats = []string{"html", "djson", "native", "text"} -func getFilePlaces(wd string, kind string) (root string, places []place.Place) { +func getFilePlaces(wd string, kind string) (root string, places []place.ManagedPlace) { root = filepath.Clean(filepath.Join(wd, "..", "testdata", kind)) - infos, err := ioutil.ReadDir(root) + entries, err := os.ReadDir(root) if err != nil { panic(err) } cdata := manager.ConnectData{Filter: &noFilter{}, Notify: nil} - for _, info := range infos { - if info.Mode().IsDir() { + for _, entry := range entries { + if entry.IsDir() { place, err := manager.Connect( - "dir://"+filepath.Join(root, info.Name()), + "dir://"+filepath.Join(root, entry.Name()), false, &cdata, ) if err != nil { panic(err) @@ -80,11 +80,11 @@ f, err := os.Open(file) if err != nil { return "", err } defer f.Close() - src, err := ioutil.ReadAll(f) + src, err := io.ReadAll(f) return string(src), err } func checkFileContent(t *testing.T, filename string, gotContent string) { t.Helper() @@ -100,47 +100,50 @@ } } func checkBlocksFile(t *testing.T, resultName string, zn *ast.ZettelNode, format string) { t.Helper() - if enc := encoder.Create(format); enc != nil { + var env encoder.Environment + if enc := encoder.Create(format, &env); enc != nil { var sb strings.Builder enc.WriteBlocks(&sb, zn.Ast) checkFileContent(t, resultName, sb.String()) return } panic(fmt.Sprintf("Unknown writer format %q", format)) } func checkZmkEncoder(t *testing.T, zn *ast.ZettelNode) { - zmkEncoder := encoder.Create("zmk") + zmkEncoder := encoder.Create("zmk", nil) var sb strings.Builder zmkEncoder.WriteBlocks(&sb, zn.Ast) gotFirst := sb.String() sb.Reset() newZettel := parser.ParseZettel(domain.Zettel{ - Meta: zn.Zettel.Meta, Content: domain.NewContent("\n" + gotFirst)}, "") + Meta: zn.Meta, Content: domain.NewContent("\n" + gotFirst)}, "") zmkEncoder.WriteBlocks(&sb, newZettel.Ast) gotSecond := sb.String() sb.Reset() if gotFirst != gotSecond { t.Errorf("\n1st: %q\n2nd: %q", gotFirst, gotSecond) } } -func getPlaceName(p place.Place, root string) string { +func getPlaceName(p place.ManagedPlace, root string) string { return p.Location()[len("dir://")+len(root):] } -func checkContentPlace(t *testing.T, p place.Place, wd, placeName string) { +func match(*meta.Meta) bool { return true } + +func checkContentPlace(t *testing.T, p place.ManagedPlace, wd, placeName string) { ss := p.(place.StartStopper) if err := ss.Start(context.Background()); err != nil { panic(err) } - metaList, err := p.SelectMeta(context.Background(), nil, nil) + metaList, err := p.SelectMeta(context.Background(), match) if err != nil { panic(err) } for _, meta := range metaList { zettel, err := p.GetZettel(context.Background(), meta.Zid) @@ -159,11 +162,10 @@ }) } if err := ss.Stop(context.Background()); err != nil { panic(err) } - } func TestContentRegression(t *testing.T) { wd, err := os.Getwd() if err != nil { @@ -176,25 +178,25 @@ } func checkMetaFile(t *testing.T, resultName string, zn *ast.ZettelNode, format string) { t.Helper() - if enc := encoder.Create(format); enc != nil { + if enc := encoder.Create(format, nil); enc != nil { var sb strings.Builder - enc.WriteMeta(&sb, zn.Zettel.Meta) + enc.WriteMeta(&sb, zn.Meta) checkFileContent(t, resultName, sb.String()) return } panic(fmt.Sprintf("Unknown writer format %q", format)) } -func checkMetaPlace(t *testing.T, p place.Place, wd, placeName string) { +func checkMetaPlace(t *testing.T, p place.ManagedPlace, wd, placeName string) { ss := p.(place.StartStopper) if err := ss.Start(context.Background()); err != nil { panic(err) } - metaList, err := p.SelectMeta(context.Background(), nil, nil) + metaList, err := p.SelectMeta(context.Background(), match) if err != nil { panic(err) } for _, meta := range metaList { zettel, err := p.GetZettel(context.Background(), meta.Zid) Index: tests/result/meta/copyright/20200310125800.djson ================================================================== --- tests/result/meta/copyright/20200310125800.djson +++ tests/result/meta/copyright/20200310125800.djson @@ -1,1 +1,1 @@ -{"title":"Header Test","role":"zettel","syntax":"zmk","copyright":"(c) 2020 Detlef Stern","license":"CC BY-SA 4.0"} +{"title":[{"t":"Text","s":"Header"},{"t":"Space"},{"t":"Text","s":"Test"}],"role":"zettel","syntax":"zmk","copyright":"(c) 2020 Detlef Stern","license":"CC BY-SA 4.0"} Index: tests/result/meta/copyright/20200310125800.native ================================================================== --- tests/result/meta/copyright/20200310125800.native +++ tests/result/meta/copyright/20200310125800.native @@ -1,6 +1,6 @@ -[Title "Header Test"] +[Title Text "Header",Space,Text "Test"] [Role "zettel"] [Syntax "zmk"] [Header [copyright "(c) 2020 Detlef Stern"], [license "CC BY-SA 4.0"]] Index: tests/result/meta/header/20200310125800.djson ================================================================== --- tests/result/meta/header/20200310125800.djson +++ tests/result/meta/header/20200310125800.djson @@ -1,1 +1,1 @@ -{"title":"Header Test","role":"zettel","syntax":"zmk","x-no":"00000000000000"} +{"title":[{"t":"Text","s":"Header"},{"t":"Space"},{"t":"Text","s":"Test"}],"role":"zettel","syntax":"zmk","x-no":"00000000000000"} Index: tests/result/meta/header/20200310125800.native ================================================================== --- tests/result/meta/header/20200310125800.native +++ tests/result/meta/header/20200310125800.native @@ -1,5 +1,5 @@ -[Title "Header Test"] +[Title Text "Header",Space,Text "Test"] [Role "zettel"] [Syntax "zmk"] [Header [x-no "00000000000000"]] Index: tests/result/meta/title/20200310110300.djson ================================================================== --- tests/result/meta/title/20200310110300.djson +++ tests/result/meta/title/20200310110300.djson @@ -1,1 +1,1 @@ -{"title":"A \"\"Title\"\" with //Markup//, ``Zettelmarkup``{=zmk}","role":"zettel","syntax":"zmk"} +{"title":[{"t":"Text","s":"A"},{"t":"Space"},{"t":"Quote","i":[{"t":"Text","s":"Title"}]},{"t":"Space"},{"t":"Text","s":"with"},{"t":"Space"},{"t":"Italic","i":[{"t":"Text","s":"Markup"}]},{"t":"Text","s":","},{"t":"Space"},{"t":"Code","a":{"":"zmk"},"s":"Zettelmarkup"}],"role":"zettel","syntax":"zmk"} Index: tests/result/meta/title/20200310110300.html ================================================================== --- tests/result/meta/title/20200310110300.html +++ tests/result/meta/title/20200310110300.html @@ -1,3 +1,3 @@ - + Index: tests/result/meta/title/20200310110300.native ================================================================== --- tests/result/meta/title/20200310110300.native +++ tests/result/meta/title/20200310110300.native @@ -1,3 +1,3 @@ -[Title "A \"\"Title\"\" with //Markup//, ``Zettelmarkup``{=zmk}"] +[Title Text "A",Space,Quote [Text "Title"],Space,Text "with",Space,Italic [Text "Markup"],Text ",",Space,Code ("zmk",[]) "Zettelmarkup"] [Role "zettel"] [Syntax "zmk"] Index: tests/result/meta/title/20200310110300.text ================================================================== --- tests/result/meta/title/20200310110300.text +++ tests/result/meta/title/20200310110300.text @@ -1,3 +1,3 @@ -A ""Title"" with //Markup//, ``Zettelmarkup``{=zmk} +A Title with Markup, Zettelmarkup zettel zmk Index: tools/build.go ================================================================== --- tools/build.go +++ tools/build.go @@ -16,11 +16,11 @@ "bytes" "errors" "flag" "fmt" "io" - "io/ioutil" + "io/fs" "os" "os/exec" "path/filepath" "regexp" "strings" @@ -48,11 +48,11 @@ err := cmd.Run() return out.String(), err } func readVersionFile() (string, error) { - content, err := ioutil.ReadFile("VERSION") + content, err := os.ReadFile("VERSION") if err != nil { return "", err } return strings.TrimFunc(string(content), func(r rune) bool { return r <= ' ' @@ -131,10 +131,13 @@ if err := checkGoLint(); err != nil { return err } if err := checkGoVetShadow(); err != nil { return err + } + if err := checkStaticcheck(); err != nil { + return err } return checkFossilExtra() } func checkGoTest() error { @@ -161,15 +164,13 @@ return err } func checkGoLint() error { out, err := executeCommand(nil, "golint", "./...") - if err != nil { + if out != "" { fmt.Fprintln(os.Stderr, "Some lints failed") - if len(out) > 0 { - fmt.Fprintln(os.Stderr, out) - } + fmt.Fprint(os.Stderr, out) } return err } func checkGoVetShadow() error { @@ -181,10 +182,20 @@ if err != nil { fmt.Fprintln(os.Stderr, "Some shadowed variables found") if len(out) > 0 { fmt.Fprintln(os.Stderr, out) } + } + return err +} +func checkStaticcheck() error { + out, err := executeCommand(nil, "staticcheck", "./...") + if err != nil { + fmt.Fprintln(os.Stderr, "Some staticcheck problems found") + if len(out) > 0 { + fmt.Fprintln(os.Stderr, out) + } } return err } func checkFossilExtra() error { @@ -227,22 +238,79 @@ fmt.Println(out) } return nil } -func cmdRelease() error { +func cmdManual() error { + base, _ := getReleaseVersionData() + return createManualZip(".", base) +} + +func createManualZip(path, base string) error { + manualPath := filepath.Join("docs", "manual") + entries, err := os.ReadDir(manualPath) + if err != nil { + return err + } + zipName := filepath.Join(path, "manual-"+base+".zip") + zipFile, err := os.OpenFile(zipName, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return err + } + defer zipFile.Close() + zipWriter := zip.NewWriter(zipFile) + defer zipWriter.Close() + + for _, entry := range entries { + if err = createManualZipEntry(manualPath, entry, zipWriter); err != nil { + return err + } + } + return nil +} + +func createManualZipEntry(path string, entry fs.DirEntry, zipWriter *zip.Writer) error { + info, err := entry.Info() + if err != nil { + return err + } + fh, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + fh.Name = entry.Name() + fh.Method = zip.Deflate + w, err := zipWriter.CreateHeader(fh) + if err != nil { + return err + } + manualFile, err := os.Open(filepath.Join(path, entry.Name())) + if err != nil { + return err + } + defer manualFile.Close() + _, err = io.Copy(w, manualFile) + return err +} + +func getReleaseVersionData() (string, string) { base, fossil := getVersionData() if strings.HasSuffix(base, "dev") { base = base[:len(base)-3] + "preview-" + time.Now().Format("20060102") } if strings.HasSuffix(fossil, dirtySuffix) { fmt.Fprintf(os.Stderr, "Warning: releasing a dirty version %v\n", fossil) base = base + dirtySuffix } + return base, fossil +} + +func cmdRelease() error { if err := cmdCheck(); err != nil { return err } + base, fossil := getReleaseVersionData() releases := []struct { arch string os string env []string name string @@ -258,45 +326,57 @@ zsName := filepath.Join("releases", rel.name) if err := doBuild(env, calcVersion(base, fossil), zsName); err != nil { return err } zipName := fmt.Sprintf("zettelstore-%v-%v-%v.zip", base, rel.os, rel.arch) - if err := createZip(zsName, zipName, rel.name); err != nil { + if err := createReleaseZip(zsName, zipName, rel.name); err != nil { return err } if err := os.Remove(zsName); err != nil { return err } } - return nil + return createManualZip("releases", base) } -func createZip(zsName, zipName, fileName string) error { - zsFile, err := os.Open(zsName) - if err != nil { - return err - } - defer zsFile.Close() +func createReleaseZip(zsName, zipName, fileName string) error { zipFile, err := os.OpenFile(filepath.Join("releases", zipName), os.O_RDWR|os.O_CREATE, 0600) if err != nil { return err } defer zipFile.Close() + zw := zip.NewWriter(zipFile) + defer zw.Close() + err = addFileToZip(zw, zsName, fileName) + if err != nil { + return err + } + err = addFileToZip(zw, "LICENSE.txt", "LICENSE.txt") + if err != nil { + return err + } + err = addFileToZip(zw, "docs/readmezip.txt", "README.txt") + return err +} +func addFileToZip(zipFile *zip.Writer, filepath, filename string) error { + zsFile, err := os.Open(filepath) + if err != nil { + return err + } + defer zsFile.Close() stat, err := zsFile.Stat() if err != nil { return err } fh, err := zip.FileInfoHeader(stat) if err != nil { return err } - fh.Name = fileName + fh.Name = filename fh.Method = zip.Deflate - zw := zip.NewWriter(zipFile) - defer zw.Close() - w, err := zw.CreateHeader(fh) + w, err := zipFile.CreateHeader(fh) if err != nil { return err } _, err = io.Copy(w, zsFile) return err @@ -323,10 +403,11 @@ check Check current working state: execute tests, static analysis tools, extra files, ... Is automatically done when releasing the software. clean Remove all build and release directories. help Outputs this text. + manual Create a ZIP file with all manual zettel release Create the software for various platforms and put them in appropriate named ZIP files. version Print the current version of the software. All commands can be abbreviated as long as they remain unique.`) @@ -345,10 +426,12 @@ cmdHelp() } else { switch args[0] { case "b", "bu", "bui", "buil", "build": err = cmdBuild() + case "m", "ma", "man", "manu", "manua", "manual": + err = cmdManual() case "r", "re", "rel", "rele", "relea", "releas", "release": err = cmdRelease() case "cl", "cle", "clea", "clean": err = cmdClean() case "v", "ve", "ver", "vers", "versi", "versio", "version": Index: usecase/authenticate.go ================================================================== --- usecase/authenticate.go +++ usecase/authenticate.go @@ -18,17 +18,17 @@ "zettelstore.de/z/auth/cred" "zettelstore.de/z/auth/token" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" - "zettelstore.de/z/place" + "zettelstore.de/z/search" ) // AuthenticatePort is the interface used by this use case. type AuthenticatePort interface { GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) - SelectMeta(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) + SelectMeta(ctx context.Context, s *search.Search) ([]*meta.Meta, error) } // Authenticate is the data for this use case. type Authenticate struct { port AuthenticatePort Index: usecase/context.go ================================================================== --- usecase/context.go +++ usecase/context.go @@ -14,19 +14,16 @@ import ( "context" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" - "zettelstore.de/z/place" ) // ZettelContextPort is the interface used by this use case. type ZettelContextPort interface { // GetMeta retrieves just the meta data of a specific zettel. GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) - - SelectMeta(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) } // ZettelContext is the data for this use case. type ZettelContext struct { port ZettelContextPort @@ -110,28 +107,10 @@ return result, nil } func (uc ZettelContext) addInitialTasks(ctx context.Context, tasks *ztlCtx, start *meta.Meta) { tasks.add(start, 0) - tags, ok := start.GetTags(meta.KeyTags) - if !ok { - return - } - filter := place.Filter{Expr: map[string][]string{}} - limit := tasks.depth - if limit == 0 || limit > 10 { - limit = 10 - } - sorter := place.Sorter{Limit: limit} - for _, tag := range tags { - filter.Expr[meta.KeyTags] = []string{tag} - if ml, err := uc.port.SelectMeta(ctx, &filter, &sorter); err == nil { - for _, m := range ml { - tasks.add(m, 1) - } - } - } } func (uc ZettelContext) addID(ctx context.Context, tasks *ztlCtx, depth int, value string) { if zid, err := id.Parse(value); err == nil { if m, err := uc.port.GetMeta(ctx, zid); err == nil { Index: usecase/get_user.go ================================================================== --- usecase/get_user.go +++ usecase/get_user.go @@ -16,20 +16,20 @@ "zettelstore.de/z/config/startup" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/index" - "zettelstore.de/z/place" + "zettelstore.de/z/search" ) // Use case: return user identified by meta key ident. // --------------------------------------------------- // GetUserPort is the interface used by this use case. type GetUserPort interface { GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) - SelectMeta(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) + SelectMeta(ctx context.Context, s *search.Search) ([]*meta.Meta, error) } // GetUser is the data for this use case. type GetUser struct { port GetUserPort @@ -57,17 +57,14 @@ return nil, nil } return identMeta, nil } // Owner was not found or has another ident. Try via list search. - filter := place.Filter{ - Expr: map[string][]string{ - meta.KeyRole: {meta.ValueRoleUser}, - meta.KeyUserID: {ident}, - }, - } - metaList, err := uc.port.SelectMeta(ctx, &filter, nil) + var s *search.Search + s = s.AddExpr(meta.KeyRole, meta.ValueRoleUser, false) + s = s.AddExpr(meta.KeyUserID, ident, false) + metaList, err := uc.port.SelectMeta(ctx, s) if err != nil { return nil, err } if len(metaList) < 1 { return nil, nil Index: usecase/list_meta.go ================================================================== --- usecase/list_meta.go +++ usecase/list_meta.go @@ -13,18 +13,17 @@ import ( "context" "zettelstore.de/z/domain/meta" - "zettelstore.de/z/place" + "zettelstore.de/z/search" ) // ListMetaPort is the interface used by this use case. type ListMetaPort interface { - // SelectMeta returns all zettel meta data that match the selection - // criteria. The result is ordered by descending zettel id. - SelectMeta(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) + // SelectMeta returns all zettel meta data that match the selection criteria. + SelectMeta(ctx context.Context, s *search.Search) ([]*meta.Meta, error) } // ListMeta is the data for this use case. type ListMeta struct { port ListMetaPort @@ -34,8 +33,8 @@ func NewListMeta(port ListMetaPort) ListMeta { return ListMeta{port: port} } // Run executes the use case. -func (uc ListMeta) Run(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) { - return uc.port.SelectMeta(ctx, f, s) +func (uc ListMeta) Run(ctx context.Context, s *search.Search) ([]*meta.Meta, error) { + return uc.port.SelectMeta(ctx, s) } Index: usecase/list_role.go ================================================================== --- usecase/list_role.go +++ usecase/list_role.go @@ -15,18 +15,17 @@ "context" "sort" "zettelstore.de/z/domain/meta" "zettelstore.de/z/index" - "zettelstore.de/z/place" + "zettelstore.de/z/search" ) // ListRolePort is the interface used by this use case. type ListRolePort interface { - // SelectMeta returns all zettel meta data that match the selection - // criteria. The result is ordered by descending zettel id. - SelectMeta(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) + // SelectMeta returns all zettel meta data that match the selection criteria. + SelectMeta(ctx context.Context, s *search.Search) ([]*meta.Meta, error) } // ListRole is the data for this use case. type ListRole struct { port ListRolePort @@ -37,11 +36,11 @@ return ListRole{port: port} } // Run executes the use case. func (uc ListRole) Run(ctx context.Context) ([]string, error) { - metas, err := uc.port.SelectMeta(index.NoEnrichContext(ctx), nil, nil) + metas, err := uc.port.SelectMeta(index.NoEnrichContext(ctx), nil) if err != nil { return nil, err } roles := make(map[string]bool, 8) for _, m := range metas { Index: usecase/list_tags.go ================================================================== --- usecase/list_tags.go +++ usecase/list_tags.go @@ -14,18 +14,17 @@ import ( "context" "zettelstore.de/z/domain/meta" "zettelstore.de/z/index" - "zettelstore.de/z/place" + "zettelstore.de/z/search" ) // ListTagsPort is the interface used by this use case. type ListTagsPort interface { - // SelectMeta returns all zettel meta data that match the selection - // criteria. The result is ordered by descending zettel id. - SelectMeta(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) + // SelectMeta returns all zettel meta data that match the selection criteria. + SelectMeta(ctx context.Context, s *search.Search) ([]*meta.Meta, error) } // ListTags is the data for this use case. type ListTags struct { port ListTagsPort @@ -39,19 +38,18 @@ // TagData associates tags with a list of all zettel meta that use this tag type TagData map[string][]*meta.Meta // Run executes the use case. func (uc ListTags) Run(ctx context.Context, minCount int) (TagData, error) { - metas, err := uc.port.SelectMeta(index.NoEnrichContext(ctx), nil, nil) + metas, err := uc.port.SelectMeta(index.NoEnrichContext(ctx), nil) if err != nil { return nil, err } result := make(TagData) for _, m := range metas { if tl, ok := m.GetList(meta.KeyTags); ok && len(tl) > 0 { for _, t := range tl { - t = meta.CleanTag(t) result[t] = append(result[t], m) } } } if minCount > 1 { Index: usecase/order.go ================================================================== --- usecase/order.go +++ usecase/order.go @@ -49,7 +49,7 @@ if m, err := uc.port.GetMeta(ctx, zid); err == nil { result = append(result, m) } } } - return zn.Zettel.Meta, result, nil + return zn.Meta, result, nil } Index: usecase/search.go ================================================================== --- usecase/search.go +++ usecase/search.go @@ -14,18 +14,17 @@ import ( "context" "zettelstore.de/z/domain/meta" "zettelstore.de/z/index" - "zettelstore.de/z/place" + "zettelstore.de/z/search" ) // SearchPort is the interface used by this use case. type SearchPort interface { - // SelectMeta returns all zettel meta data that match the selection - // criteria. The result is ordered by descending zettel id. - SelectMeta(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) + // SelectMeta returns all zettel meta data that match the selection criteria. + SelectMeta(ctx context.Context, s *search.Search) ([]*meta.Meta, error) } // Search is the data for this use case. type Search struct { port SearchPort @@ -35,28 +34,11 @@ func NewSearch(port SearchPort) Search { return Search{port: port} } // Run executes the use case. -func (uc Search) Run(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) { - // TODO: interpret f.Expr[""]. Can contain expressions for specific meta tags. - if !usesComputedMeta(f, s) { +func (uc Search) Run(ctx context.Context, s *search.Search) ([]*meta.Meta, error) { + if !s.HasComputedMetaKey() { ctx = index.NoEnrichContext(ctx) } - return uc.port.SelectMeta(ctx, f, s) -} - -func usesComputedMeta(f *place.Filter, s *place.Sorter) bool { - if f != nil { - for key := range f.Expr { - if key == "" || meta.IsComputed(key) { - return true - } - } - } - if s != nil { - if order := s.Order; order != "" && meta.IsComputed(order) { - return true - } - } - return false + return uc.port.SelectMeta(ctx, s) } Index: web/adapter/api/get_links.go ================================================================== --- web/adapter/api/get_links.go +++ web/adapter/api/get_links.go @@ -66,36 +66,14 @@ outData := jsonGetLinks{ ID: zid.String(), URL: adapter.NewURLBuilder('z').SetZid(zid).String(), } if kind&kindLink != 0 { - if matter&matterIncoming != 0 { - // Backlinks not yet implemented - outData.Links.Incoming = []jsonIDURL{} - } - zetRefs, locRefs, extRefs := collect.DivideReferences(summary.Links, false) - if matter&matterOutgoing != 0 { - outData.Links.Outgoing = idURLRefs(zetRefs) - } - if matter&matterLocal != 0 { - outData.Links.Local = stringRefs(locRefs) - } - if matter&matterExternal != 0 { - outData.Links.External = stringRefs(extRefs) - } + setupLinkJSONRefs(summary, matter, &outData) } if kind&kindImage != 0 { - zetRefs, locRefs, extRefs := collect.DivideReferences(summary.Images, false) - if matter&matterOutgoing != 0 { - outData.Images.Outgoing = idURLRefs(zetRefs) - } - if matter&matterLocal != 0 { - outData.Images.Local = stringRefs(locRefs) - } - if matter&matterExternal != 0 { - outData.Images.External = stringRefs(extRefs) - } + setupImageJSONRefs(summary, matter, &outData) } if kind&kindCite != 0 { outData.Cites = stringCites(summary.Cites) } @@ -103,10 +81,39 @@ enc := json.NewEncoder(w) enc.SetEscapeHTML(false) enc.Encode(&outData) } } + +func setupLinkJSONRefs(summary collect.Summary, matter matterType, outData *jsonGetLinks) { + if matter&matterIncoming != 0 { + outData.Links.Incoming = []jsonIDURL{} + } + zetRefs, locRefs, extRefs := collect.DivideReferences(summary.Links, false) + if matter&matterOutgoing != 0 { + outData.Links.Outgoing = idURLRefs(zetRefs) + } + if matter&matterLocal != 0 { + outData.Links.Local = stringRefs(locRefs) + } + if matter&matterExternal != 0 { + outData.Links.External = stringRefs(extRefs) + } +} + +func setupImageJSONRefs(summary collect.Summary, matter matterType, outData *jsonGetLinks) { + zetRefs, locRefs, extRefs := collect.DivideReferences(summary.Images, false) + if matter&matterOutgoing != 0 { + outData.Images.Outgoing = idURLRefs(zetRefs) + } + if matter&matterLocal != 0 { + outData.Images.Local = stringRefs(locRefs) + } + if matter&matterExternal != 0 { + outData.Images.External = stringRefs(extRefs) + } +} func idURLRefs(refs []*ast.Reference) []jsonIDURL { result := make([]jsonIDURL, 0, len(refs)) for _, ref := range refs { path := ref.URL.Path Index: web/adapter/api/get_zettel.go ================================================================== --- web/adapter/api/get_zettel.go +++ web/adapter/api/get_zettel.go @@ -63,62 +63,50 @@ adapter.InternalServerError(w, "Write D/JSON", err) } return } - langOption := encoder.StringOption{Key: "lang", Value: runtime.GetLang(zn.InhMeta)} - linkAdapter := encoder.AdaptLinkOption{ - Adapter: adapter.MakeLinkAdapter(ctx, 'z', getMeta, part.DefString(partZettel), format), + env := encoder.Environment{ + LinkAdapter: adapter.MakeLinkAdapter(ctx, 'z', getMeta, part.DefString(partZettel), format), + ImageAdapter: adapter.MakeImageAdapter(), + CiteAdapter: nil, + Lang: runtime.GetLang(zn.InhMeta), + Xhtml: false, + MarkerExternal: "", + NewWindow: false, + IgnoreMeta: map[string]bool{meta.KeyLang: true}, } - imageAdapter := encoder.AdaptImageOption{Adapter: adapter.MakeImageAdapter()} - switch part { case partZettel: inhMeta := false if format != "raw" { w.Header().Set(adapter.ContentType, format2ContentType(format)) inhMeta = true } - enc := encoder.Create(format, &langOption, - &linkAdapter, - &imageAdapter, - &encoder.StringsOption{ - Key: "no-meta", - Value: []string{ - meta.KeyLang, - }, - }, - ) + enc := encoder.Create(format, &env) if enc == nil { err = adapter.ErrNoSuchFormat } else { _, err = enc.WriteZettel(w, zn, inhMeta) } case partMeta: w.Header().Set(adapter.ContentType, format2ContentType(format)) if format == "raw" { // Don't write inherited meta data, just the raw - err = writeMeta(w, zn.Zettel.Meta, format) + err = writeMeta(w, zn.Meta, format, nil) } else { - err = writeMeta(w, zn.InhMeta, format) + err = writeMeta(w, zn.InhMeta, format, nil) } case partContent: if format == "raw" { - if ct, ok := syntax2contentType(runtime.GetSyntax(zn.Zettel.Meta)); ok { + if ct, ok := syntax2contentType(runtime.GetSyntax(zn.Meta)); ok { w.Header().Add(adapter.ContentType, ct) } } else { w.Header().Set(adapter.ContentType, format2ContentType(format)) } - err = writeContent(w, zn, format, - &langOption, - &encoder.StringOption{ - Key: meta.KeyMarkerExternal, - Value: runtime.GetMarkerExternal()}, - &linkAdapter, - &imageAdapter, - ) + err = writeContent(w, zn, format, &env) default: adapter.BadRequest(w, "Unknown _part parameter") return } if err != nil { Index: web/adapter/api/get_zettel_list.go ================================================================== --- web/adapter/api/get_zettel_list.go +++ web/adapter/api/get_zettel_list.go @@ -31,18 +31,18 @@ parseZettel usecase.ParseZettel, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() q := r.URL.Query() - filter, sorter := adapter.GetFilterSorter(q, false) + s := adapter.GetSearch(q, false) format := adapter.GetFormat(r, q, encoder.GetDefaultFormat()) part := getPart(q, partMeta) ctx1 := ctx - if format == "html" || (filter == nil && sorter == nil && (part == partID || part == partContent)) { + if format == "html" || (!s.HasComputedMetaKey() && (part == partID || part == partContent)) { ctx1 = index.NoEnrichContext(ctx1) } - metaList, err := listMeta.Run(ctx1, filter, sorter) + metaList, err := listMeta.Run(ctx1, s) if err != nil { adapter.ReportUsecaseError(w, err) return } @@ -59,25 +59,25 @@ } } } func renderListMetaHTML(w http.ResponseWriter, metaList []*meta.Meta) { + env := encoder.Environment{Interactive: true} buf := encoder.NewBufWriter(w) - buf.WriteStrings("\n\n
      \n") for _, m := range metaList { title := m.GetDefault(meta.KeyTitle, "") - htmlTitle, err := adapter.FormatInlines(parser.ParseTitle(title), "html") + htmlTitle, err := adapter.FormatInlines(parser.ParseMetadata(title), "html", &env) if err != nil { adapter.InternalServerError(w, "Format HTML inlines", err) return } buf.WriteStrings( "
    • ", htmlTitle, "
      • \n") } buf.WriteString("
    \n\n") buf.Flush() } Index: web/adapter/api/json.go ================================================================== --- web/adapter/api/json.go +++ web/adapter/api/json.go @@ -21,11 +21,10 @@ "zettelstore.de/z/config/runtime" "zettelstore.de/z/domain" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/encoder" - "zettelstore.de/z/parser" "zettelstore.de/z/usecase" "zettelstore.de/z/web/adapter" ) type jsonIDURL struct { @@ -64,11 +63,11 @@ URL: adapter.NewURLBuilder('z').SetZid(z.Zid).String(), } switch part { case partZettel: - encoding, content := encodedContent(z.Zettel.Content) + encoding, content := encodedContent(z.Content) outData = jsonZettel{ ID: idData.ID, URL: idData.URL, Meta: z.InhMeta.Map(), Encoding: encoding, @@ -79,11 +78,11 @@ ID: idData.ID, URL: idData.URL, Meta: z.InhMeta.Map(), } case partContent: - encoding, content := encodedContent(z.Zettel.Content) + encoding, content := encodedContent(z.Content) outData = jsonContent{ ID: idData.ID, URL: idData.URL, Encoding: encoding, Content: content, @@ -174,11 +173,11 @@ } func writeDJSONMeta(w io.Writer, z *ast.ZettelNode) error { _, err := w.Write(djsonMetaHeader) if err == nil { - err = writeMeta(w, z.InhMeta, "djson", &encoder.TitleOption{Inline: z.Title}) + err = writeMeta(w, z.InhMeta, "djson", nil) } return err } func writeDJSONContent( @@ -188,16 +187,13 @@ part, defPart partType, getMeta usecase.GetMeta, ) (err error) { _, err = w.Write(djsonContentHeader) if err == nil { - err = writeContent(w, z, "djson", - &encoder.AdaptLinkOption{ - Adapter: adapter.MakeLinkAdapter(ctx, 'z', getMeta, part.DefString(defPart), "djson"), - }, - &encoder.AdaptImageOption{Adapter: adapter.MakeImageAdapter()}, - ) + err = writeContent(w, z, "djson", &encoder.Environment{ + LinkAdapter: adapter.MakeLinkAdapter(ctx, 'z', getMeta, part.DefString(defPart), "djson"), + ImageAdapter: adapter.MakeImageAdapter()}) } return err } var ( @@ -247,16 +243,15 @@ break } zn = z } else { zn = &ast.ZettelNode{ - Zettel: domain.Zettel{Meta: m, Content: ""}, + Meta: m, + Content: "", Zid: m.Zid, InhMeta: runtime.AddDefaultValues(m), - Title: parser.ParseTitle( - m.GetDefault(meta.KeyTitle, runtime.GetDefaultTitle())), - Ast: nil, + Ast: nil, } } if isJSON { err = writeJSONZettel(w, zn, part) } else { @@ -270,23 +265,23 @@ adapter.InternalServerError(w, "Get list", err) } } func writeContent( - w io.Writer, zn *ast.ZettelNode, format string, options ...encoder.Option) error { - enc := encoder.Create(format, options...) + w io.Writer, zn *ast.ZettelNode, format string, env *encoder.Environment) error { + enc := encoder.Create(format, env) if enc == nil { return adapter.ErrNoSuchFormat } _, err := enc.WriteContent(w, zn) return err } func writeMeta( - w io.Writer, m *meta.Meta, format string, options ...encoder.Option) error { - enc := encoder.Create(format, options...) + w io.Writer, m *meta.Meta, format string, env *encoder.Environment) error { + enc := encoder.Create(format, env) if enc == nil { return adapter.ErrNoSuchFormat } _, err := enc.WriteMeta(w, m) Index: web/adapter/encoding.go ================================================================== --- web/adapter/encoding.go +++ web/adapter/encoding.go @@ -27,12 +27,12 @@ // ErrNoSuchFormat signals an unsupported encoding format var ErrNoSuchFormat = errors.New("no such format") // FormatInlines returns a string representation of the inline slice. -func FormatInlines(is ast.InlineSlice, format string, options ...encoder.Option) (string, error) { - enc := encoder.Create(format, options...) +func FormatInlines(is ast.InlineSlice, format string, env *encoder.Environment) (string, error) { + enc := encoder.Create(format, env) if enc == nil { return "", ErrNoSuchFormat } var content strings.Builder @@ -68,40 +68,44 @@ zid, err := id.Parse(origRef.URL.Path) if err != nil { panic(err) } _, err = getMeta.Run(index.NoEnrichContext(ctx), zid) - newLink := *origLink - if err == nil { - u := NewURLBuilder(key).SetZid(zid) - if part != "" { - u.AppendQuery("_part", part) - } - if format != "" { - u.AppendQuery("_format", format) - } - if fragment := origRef.URL.EscapedFragment(); len(fragment) > 0 { - u.SetFragment(fragment) - } - newRef := ast.ParseReference(u.String()) - newRef.State = ast.RefStateFound - newLink.Ref = newRef - return &newLink - } if place.IsErrNotAllowed(err) { return &ast.FormatNode{ Code: ast.FormatSpan, Attrs: origLink.Attrs, Inlines: origLink.Inlines, } } - newRef := ast.ParseReference(origRef.Value) - newRef.State = ast.RefStateBroken + var newRef *ast.Reference + if err == nil { + newRef = ast.ParseReference(adaptZettelReference(key, zid, part, format, origRef.URL.EscapedFragment())) + newRef.State = ast.RefStateFound + } else { + newRef = ast.ParseReference(origRef.Value) + newRef.State = ast.RefStateBroken + } + newLink := *origLink newLink.Ref = newRef return &newLink } } + +func adaptZettelReference(key byte, zid id.Zid, part, format, fragment string) string { + u := NewURLBuilder(key).SetZid(zid) + if part != "" { + u.AppendQuery("_part", part) + } + if format != "" { + u.AppendQuery("_format", format) + } + if fragment != "" { + u.SetFragment(fragment) + } + return u.String() +} // MakeImageAdapter creates an adapter to change an image node during encoding. func MakeImageAdapter() func(*ast.ImageNode) ast.InlineNode { return func(origImage *ast.ImageNode) ast.InlineNode { if origImage.Ref == nil || origImage.Ref.State != ast.RefStateZettel { Index: web/adapter/request.go ================================================================== --- web/adapter/request.go +++ web/adapter/request.go @@ -16,11 +16,11 @@ "net/url" "strconv" "strings" "zettelstore.de/z/domain/meta" - "zettelstore.de/z/place" + "zettelstore.de/z/search" ) // GetInteger returns the integer value of the named query key. func GetInteger(q url.Values, key string) (int, bool) { s := q.Get(key) @@ -70,73 +70,80 @@ // TODO: only check before first ';' format, ok := mapCT2format[contentType] return format, ok } -// GetFilterSorter retrieves the specified filter and sorting options from a query. -func GetFilterSorter(q url.Values, forSearch bool) (filter *place.Filter, sorter *place.Sorter) { +// GetSearch retrieves the specified filter and sorting options from a query. +func GetSearch(q url.Values, forSearch bool) (s *search.Search) { sortQKey, orderQKey, offsetQKey, limitQKey, negateQKey, sQKey := getQueryKeys(forSearch) for key, values := range q { switch key { case sortQKey, orderQKey: - if len(values) > 0 { - descending := false - sortkey := values[0] - if strings.HasPrefix(sortkey, "-") { - descending = true - sortkey = sortkey[1:] - } - if meta.KeyIsValid(sortkey) || sortkey == place.RandomOrder { - sorter = place.EnsureSorter(sorter) - sorter.Order = sortkey - sorter.Descending = descending - } - } - case offsetQKey: - if len(values) > 0 { - if offset, err := strconv.Atoi(values[0]); err == nil { - sorter = place.EnsureSorter(sorter) - sorter.Offset = offset - } - } - case limitQKey: - if len(values) > 0 { - if limit, err := strconv.Atoi(values[0]); err == nil { - sorter = place.EnsureSorter(sorter) - sorter.Limit = limit - } - } - case negateQKey: - filter = place.EnsureFilter(filter) - filter.Negate = true - case sQKey: - if vals := cleanQueryValues(values); len(vals) > 0 { - filter = place.EnsureFilter(filter) - filter.Expr[""] = vals - } - default: - if !forSearch && meta.KeyIsValid(key) { - filter = place.EnsureFilter(filter) - filter.Expr[key] = cleanQueryValues(values) - } - } - } - return filter, sorter + s = extractOrderFromQuery(values, s) + case offsetQKey: + s = extractOffsetFromQuery(values, s) + case limitQKey: + s = extractLimitFromQuery(values, s) + case negateQKey: + s = s.SetNegate() + case sQKey: + s = setCleanedQueryValues(s, "", values) + default: + if !forSearch && meta.KeyIsValid(key) { + s = setCleanedQueryValues(s, key, values) + } + } + } + return s +} + +func extractOrderFromQuery(values []string, s *search.Search) *search.Search { + if len(values) > 0 { + descending := false + sortkey := values[0] + if strings.HasPrefix(sortkey, "-") { + descending = true + sortkey = sortkey[1:] + } + if meta.KeyIsValid(sortkey) || sortkey == search.RandomOrder { + s = s.AddOrder(sortkey, descending) + } + } + return s +} + +func extractOffsetFromQuery(values []string, s *search.Search) *search.Search { + if len(values) > 0 { + if offset, err := strconv.Atoi(values[0]); err == nil { + s = s.SetOffset(offset) + } + } + return s +} + +func extractLimitFromQuery(values []string, s *search.Search) *search.Search { + if len(values) > 0 { + if limit, err := strconv.Atoi(values[0]); err == nil { + s = s.SetLimit(limit) + } + } + return s } func getQueryKeys(forSearch bool) (string, string, string, string, string, string) { if forSearch { return "sort", "order", "offset", "limit", "negate", "s" } return "_sort", "_order", "_offset", "_limit", "_negate", "_s" } -func cleanQueryValues(values []string) []string { - result := make([]string, 0, len(values)) +func setCleanedQueryValues(filter *search.Search, key string, values []string) *search.Search { for _, val := range values { val = strings.TrimSpace(val) - if len(val) > 0 { - result = append(result, val) + if len(val) > 0 && val[0] == '!' { + filter = filter.AddExpr(key, val[1:], true) + } else { + filter = filter.AddExpr(key, val, false) } } - return result + return filter } Index: web/adapter/response.go ================================================================== --- web/adapter/response.go +++ web/adapter/response.go @@ -11,35 +11,53 @@ // Package adapter provides handlers for web requests. package adapter import ( "fmt" + "log" "net/http" "zettelstore.de/z/place" "zettelstore.de/z/usecase" ) // ReportUsecaseError returns an appropriate HTTP status code for errors in use cases. func ReportUsecaseError(w http.ResponseWriter, err error) { - if err == place.ErrNotFound { - NotFound(w, http.StatusText(404)) - return - } - if err, ok := err.(*place.ErrNotAllowed); ok { - Forbidden(w, err.Error()) - return - } - if err, ok := err.(*place.ErrInvalidID); ok { - BadRequest(w, fmt.Sprintf("Zettel-ID %q not appropriate in this context.", err.Zid.String())) - return - } - if err, ok := err.(*usecase.ErrZidInUse); ok { - BadRequest(w, fmt.Sprintf("Zettel-ID %q already in use.", err.Zid.String())) - return + code, text := CodeMessageFromError(err) + if code == http.StatusInternalServerError { + log.Printf("%v: %v", text, err) + } + http.Error(w, text, code) +} + +// ErrBadRequest is returned if the caller made an invalid HTTP request. +type ErrBadRequest struct { + Text string +} + +// NewErrBadRequest creates an new bad request error. +func NewErrBadRequest(text string) error { return &ErrBadRequest{Text: text} } + +func (err *ErrBadRequest) Error() string { return err.Text } + +// CodeMessageFromError returns an appropriate HTTP status code and text from a given error. +func CodeMessageFromError(err error) (int, string) { + if err == place.ErrNotFound { + return http.StatusNotFound, http.StatusText(http.StatusNotFound) + } + if err1, ok := err.(*place.ErrNotAllowed); ok { + return http.StatusForbidden, err1.Error() + } + if err1, ok := err.(*place.ErrInvalidID); ok { + return http.StatusBadRequest, fmt.Sprintf("Zettel-ID %q not appropriate in this context.", err1.Zid) + } + if err1, ok := err.(*usecase.ErrZidInUse); ok { + return http.StatusBadRequest, fmt.Sprintf("Zettel-ID %q already in use.", err1.Zid) + } + if err1, ok := err.(*ErrBadRequest); ok { + return http.StatusBadRequest, err1.Text } if err == place.ErrStopped { - InternalServerError(w, "Zettelstore not operational.", err) - return + return http.StatusInternalServerError, fmt.Sprintf("Zettelstore not operational: %v.", err) } - InternalServerError(w, "", err) + return http.StatusInternalServerError, err.Error() } Index: web/adapter/webui/create_zettel.go ================================================================== --- web/adapter/webui/create_zettel.go +++ web/adapter/webui/create_zettel.go @@ -6,14 +6,15 @@ // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( + "context" "fmt" "net/http" "zettelstore.de/z/config/runtime" "zettelstore.de/z/domain" @@ -21,10 +22,11 @@ "zettelstore.de/z/domain/meta" "zettelstore.de/z/encoder" "zettelstore.de/z/index" "zettelstore.de/z/input" "zettelstore.de/z/parser" + "zettelstore.de/z/place" "zettelstore.de/z/usecase" "zettelstore.de/z/web/adapter" "zettelstore.de/z/web/session" ) @@ -34,14 +36,17 @@ te *TemplateEngine, getZettel usecase.GetZettel, copyZettel usecase.CopyZettel, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - if origZettel, ok := getOrigZettel(w, r, getZettel, "Copy"); ok { - renderZettelForm(w, r, te, - copyZettel.Run(origZettel), "Copy Zettel", "Copy Zettel") + ctx := r.Context() + origZettel, err := getOrigZettel(ctx, w, r, getZettel, "Copy") + if err != nil { + te.reportError(ctx, w, err) + return } + renderZettelForm(w, r, te, copyZettel.Run(origZettel), "Copy Zettel", "Copy Zettel") } } // MakeGetFolgeZettelHandler creates a new HTTP handler to display the // HTML edit view of a follow-up zettel. @@ -49,14 +54,17 @@ te *TemplateEngine, getZettel usecase.GetZettel, folgeZettel usecase.FolgeZettel, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - if origZettel, ok := getOrigZettel(w, r, getZettel, "Folge"); ok { - renderZettelForm(w, r, te, - folgeZettel.Run(origZettel), "Folge Zettel", "Folgezettel") + ctx := r.Context() + origZettel, err := getOrigZettel(ctx, w, r, getZettel, "Folge") + if err != nil { + te.reportError(ctx, w, err) + return } + renderZettelForm(w, r, te, folgeZettel.Run(origZettel), "Folge Zettel", "Folgezettel") } } // MakeGetNewZettelHandler creates a new HTTP handler to display the // HTML edit view of a zettel. @@ -64,51 +72,53 @@ te *TemplateEngine, getZettel usecase.GetZettel, newZettel usecase.NewZettel, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - if origZettel, ok := getOrigZettel(w, r, getZettel, "New"); ok { - m := origZettel.Meta - title := parser.ParseInlines( - input.NewInput(runtime.GetTitle(m)), meta.ValueSyntaxZmk) - langOption := encoder.StringOption{Key: "lang", Value: runtime.GetLang(m)} - textTitle, err := adapter.FormatInlines(title, "text", &langOption) - if err != nil { - adapter.InternalServerError(w, "Format Text inlines for WebUI", err) - return - } - htmlTitle, err := adapter.FormatInlines(title, "html", &langOption) - if err != nil { - adapter.InternalServerError(w, "Format HTML inlines for WebUI", err) - return - } - renderZettelForm(w, r, te, newZettel.Run(origZettel), textTitle, htmlTitle) - } + ctx := r.Context() + origZettel, err := getOrigZettel(ctx, w, r, getZettel, "New") + if err != nil { + te.reportError(ctx, w, err) + return + } + m := origZettel.Meta + title := parser.ParseInlines(input.NewInput(runtime.GetTitle(m)), meta.ValueSyntaxZmk) + textTitle, err := adapter.FormatInlines(title, "text", nil) + if err != nil { + te.reportError(ctx, w, err) + return + } + env := encoder.Environment{Lang: runtime.GetLang(m)} + htmlTitle, err := adapter.FormatInlines(title, "html", &env) + if err != nil { + te.reportError(ctx, w, err) + return + } + renderZettelForm(w, r, te, newZettel.Run(origZettel), textTitle, htmlTitle) } } func getOrigZettel( + ctx context.Context, w http.ResponseWriter, r *http.Request, getZettel usecase.GetZettel, op string, -) (domain.Zettel, bool) { +) (domain.Zettel, error) { if format := adapter.GetFormat(r, r.URL.Query(), "html"); format != "html" { - adapter.BadRequest(w, fmt.Sprintf("%v zettel not possible in format %q", op, format)) - return domain.Zettel{}, false + return domain.Zettel{}, adapter.NewErrBadRequest( + fmt.Sprintf("%v zettel not possible in format %q", op, format)) } zid, err := id.Parse(r.URL.Path[1:]) if err != nil { - http.NotFound(w, r) - return domain.Zettel{}, false + return domain.Zettel{}, place.ErrNotFound } - origZettel, err := getZettel.Run(index.NoEnrichContext(r.Context()), zid) + origZettel, err := getZettel.Run(index.NoEnrichContext(ctx), zid) if err != nil { - http.NotFound(w, r) - return domain.Zettel{}, false + return domain.Zettel{}, place.ErrNotFound } - return origZettel, true + return origZettel, nil } func renderZettelForm( w http.ResponseWriter, r *http.Request, @@ -133,24 +143,26 @@ }) } // MakePostCreateZettelHandler creates a new HTTP handler to store content of // an existing zettel. -func MakePostCreateZettelHandler(createZettel usecase.CreateZettel) http.HandlerFunc { +func MakePostCreateZettelHandler(te *TemplateEngine, createZettel usecase.CreateZettel) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() zettel, hasContent, err := parseZettelForm(r, id.Invalid) if err != nil { - adapter.BadRequest(w, "Unable to read form data") + te.reportError(ctx, w, adapter.NewErrBadRequest("Unable to read form data")) return } if !hasContent { - adapter.BadRequest(w, "Content is missing") + te.reportError(ctx, w, adapter.NewErrBadRequest("Content is missing")) return } - if newZid, err := createZettel.Run(r.Context(), zettel); err != nil { - adapter.ReportUsecaseError(w, err) - } else { - http.Redirect(w, r, adapter.NewURLBuilder('h').SetZid(newZid).String(), http.StatusFound) + newZid, err := createZettel.Run(r.Context(), zettel) + if err != nil { + te.reportError(ctx, w, err) + return } + redirectFound(w, r, adapter.NewURLBuilder('h').SetZid(newZid)) } } Index: web/adapter/webui/delete_zettel.go ================================================================== --- web/adapter/webui/delete_zettel.go +++ web/adapter/webui/delete_zettel.go @@ -1,25 +1,26 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "fmt" "net/http" "zettelstore.de/z/config/runtime" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" + "zettelstore.de/z/place" "zettelstore.de/z/usecase" "zettelstore.de/z/web/adapter" "zettelstore.de/z/web/session" ) @@ -28,25 +29,26 @@ func MakeGetDeleteZettelHandler( te *TemplateEngine, getZettel usecase.GetZettel, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() if format := adapter.GetFormat(r, r.URL.Query(), "html"); format != "html" { - adapter.BadRequest(w, fmt.Sprintf("Delete zettel not possible in format %q", format)) + te.reportError(ctx, w, adapter.NewErrBadRequest( + fmt.Sprintf("Delete zettel not possible in format %q", format))) return } zid, err := id.Parse(r.URL.Path[1:]) if err != nil { - http.NotFound(w, r) + te.reportError(ctx, w, place.ErrNotFound) return } - ctx := r.Context() zettel, err := getZettel.Run(ctx, zid) if err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } user := session.GetUser(ctx) m := zettel.Meta @@ -61,20 +63,21 @@ }) } } // MakePostDeleteZettelHandler creates a new HTTP handler to delete a zettel. -func MakePostDeleteZettelHandler(deleteZettel usecase.DeleteZettel) http.HandlerFunc { +func MakePostDeleteZettelHandler(te *TemplateEngine, deleteZettel usecase.DeleteZettel) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() zid, err := id.Parse(r.URL.Path[1:]) if err != nil { - http.NotFound(w, r) + te.reportError(ctx, w, place.ErrNotFound) return } if err := deleteZettel.Run(r.Context(), zid); err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } - http.Redirect(w, r, adapter.NewURLBuilder('/').String(), http.StatusFound) + redirectFound(w, r, adapter.NewURLBuilder('/')) } } Index: web/adapter/webui/edit_zettel.go ================================================================== --- web/adapter/webui/edit_zettel.go +++ web/adapter/webui/edit_zettel.go @@ -6,11 +6,11 @@ // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "fmt" "net/http" @@ -17,10 +17,11 @@ "zettelstore.de/z/config/runtime" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/index" + "zettelstore.de/z/place" "zettelstore.de/z/usecase" "zettelstore.de/z/web/adapter" "zettelstore.de/z/web/session" ) @@ -27,25 +28,26 @@ // MakeEditGetZettelHandler creates a new HTTP handler to display the // HTML edit view of a zettel. func MakeEditGetZettelHandler( te *TemplateEngine, getZettel usecase.GetZettel) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() zid, err := id.Parse(r.URL.Path[1:]) if err != nil { - http.NotFound(w, r) + te.reportError(ctx, w, place.ErrNotFound) return } - ctx := r.Context() zettel, err := getZettel.Run(index.NoEnrichContext(ctx), zid) if err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } if format := adapter.GetFormat(r, r.URL.Query(), "html"); format != "html" { - adapter.BadRequest(w, fmt.Sprintf("Edit zettel %q not possible in format %q", zid.String(), format)) + te.reportError(ctx, w, adapter.NewErrBadRequest( + fmt.Sprintf("Edit zettel %q not possible in format %q", zid, format))) return } user := session.GetUser(ctx) m := zettel.Meta @@ -64,26 +66,27 @@ } } // MakeEditSetZettelHandler creates a new HTTP handler to store content of // an existing zettel. -func MakeEditSetZettelHandler(updateZettel usecase.UpdateZettel) http.HandlerFunc { +func MakeEditSetZettelHandler(te *TemplateEngine, updateZettel usecase.UpdateZettel) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() zid, err := id.Parse(r.URL.Path[1:]) if err != nil { - http.NotFound(w, r) + te.reportError(ctx, w, place.ErrNotFound) return } + zettel, hasContent, err := parseZettelForm(r, zid) if err != nil { - adapter.BadRequest(w, "Unable to read zettel form") + te.reportError(ctx, w, adapter.NewErrBadRequest("Unable to read zettel form")) return } if err := updateZettel.Run(r.Context(), zettel, hasContent); err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } - http.Redirect( - w, r, adapter.NewURLBuilder('h').SetZid(zid).String(), http.StatusFound) + redirectFound(w, r, adapter.NewURLBuilder('h').SetZid(zid)) } } Index: web/adapter/webui/forms.go ================================================================== --- web/adapter/webui/forms.go +++ web/adapter/webui/forms.go @@ -1,16 +1,16 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "net/http" "strings" Index: web/adapter/webui/get_info.go ================================================================== --- web/adapter/webui/get_info.go +++ web/adapter/webui/get_info.go @@ -6,11 +6,11 @@ // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "fmt" "net/http" @@ -18,11 +18,14 @@ "zettelstore.de/z/ast" "zettelstore.de/z/collect" "zettelstore.de/z/config/runtime" "zettelstore.de/z/domain/id" + "zettelstore.de/z/domain/meta" "zettelstore.de/z/encoder" + "zettelstore.de/z/encoder/encfun" + "zettelstore.de/z/place" "zettelstore.de/z/usecase" "zettelstore.de/z/web/adapter" "zettelstore.de/z/web/session" ) @@ -45,72 +48,53 @@ te *TemplateEngine, parseZettel usecase.ParseZettel, getMeta usecase.GetMeta, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() q := r.URL.Query() if format := adapter.GetFormat(r, q, "html"); format != "html" { - adapter.BadRequest(w, fmt.Sprintf("Zettel info not available in format %q", format)) + te.reportError(ctx, w, adapter.NewErrBadRequest( + fmt.Sprintf("Zettel info not available in format %q", format))) return } zid, err := id.Parse(r.URL.Path[1:]) if err != nil { - http.NotFound(w, r) + te.reportError(ctx, w, place.ErrNotFound) return } - ctx := r.Context() zn, err := parseZettel.Run(ctx, zid, q.Get("syntax")) if err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } - langOption := &encoder.StringOption{ - Key: "lang", - Value: runtime.GetLang(zn.InhMeta)} - summary := collect.References(zn) locLinks, extLinks := splitLocExtLinks(append(summary.Links, summary.Images...)) - textTitle, err := adapter.FormatInlines(zn.Title, "text", nil, langOption) - if err != nil { - adapter.InternalServerError(w, "Format Text inlines for info", err) - return - } - - pairs := zn.Zettel.Meta.Pairs(true) - metaData := make([]metaDataInfo, 0, len(pairs)) - getTitle := makeGetTitle(ctx, getMeta, langOption) - for _, p := range pairs { - var html strings.Builder - writeHTMLMetaValue(&html, zn.Zettel.Meta, p.Key, getTitle, langOption) - metaData = append(metaData, metaDataInfo{p.Key, html.String()}) - } - formats := encoder.GetFormats() - defFormat := encoder.GetDefaultFormat() - parts := []string{"zettel", "meta", "content"} - matrix := make([]matrixLine, 0, len(parts)) - u := adapter.NewURLBuilder('z').SetZid(zid) - for _, part := range parts { - row := make([]matrixElement, 0, len(formats)+1) - row = append(row, matrixElement{part, false, ""}) - for _, format := range formats { - u.AppendQuery("_part", part) - if format != defFormat { - u.AppendQuery("_format", format) - } - row = append(row, matrixElement{format, true, u.String()}) - u.ClearQuery() - } - matrix = append(matrix, matrixLine{row}) - } - user := session.GetUser(ctx) - var base baseData - te.makeBaseData(ctx, langOption.Value, textTitle, user, &base) - canCopy := base.CanCreate && !zn.Zettel.Content.IsBinary() + lang := runtime.GetLang(zn.InhMeta) + env := encoder.Environment{Lang: lang} + pairs := zn.Meta.Pairs(true) + metaData := make([]metaDataInfo, len(pairs)) + getTitle := makeGetTitle(ctx, getMeta, &env) + for i, p := range pairs { + var html strings.Builder + writeHTMLMetaValue(&html, zn.Meta, p.Key, getTitle, &env) + metaData[i] = metaDataInfo{p.Key, html.String()} + } + endnotes, err := formatBlocks(nil, "html", &env) + if err != nil { + endnotes = "" + } + + textTitle := encfun.MetaAsText(zn.InhMeta, meta.KeyTitle) + user := session.GetUser(ctx) + var base baseData + te.makeBaseData(ctx, lang, textTitle, user, &base) + canCopy := base.CanCreate && !zn.Content.IsBinary() te.renderTemplate(ctx, w, id.InfoTemplateZid, &base, struct { Zid string WebURL string ContextURL string CanWrite bool @@ -129,32 +113,34 @@ LocLinks []string HasExtLinks bool ExtLinks []string ExtNewWindow string Matrix []matrixLine + Endnotes string }{ Zid: zid.String(), WebURL: adapter.NewURLBuilder('h').SetZid(zid).String(), ContextURL: adapter.NewURLBuilder('j').SetZid(zid).String(), - CanWrite: te.canWrite(ctx, user, zn.Zettel), + CanWrite: te.canWrite(ctx, user, zn.Meta, zn.Content), EditURL: adapter.NewURLBuilder('e').SetZid(zid).String(), - CanFolge: base.CanCreate && !zn.Zettel.Content.IsBinary(), + CanFolge: base.CanCreate && !zn.Content.IsBinary(), FolgeURL: adapter.NewURLBuilder('f').SetZid(zid).String(), CanCopy: canCopy, CopyURL: adapter.NewURLBuilder('c').SetZid(zid).String(), - CanRename: te.canRename(ctx, user, zn.Zettel.Meta), + CanRename: te.canRename(ctx, user, zn.Meta), RenameURL: adapter.NewURLBuilder('b').SetZid(zid).String(), - CanDelete: te.canDelete(ctx, user, zn.Zettel.Meta), + CanDelete: te.canDelete(ctx, user, zn.Meta), DeleteURL: adapter.NewURLBuilder('d').SetZid(zid).String(), MetaData: metaData, HasLinks: len(extLinks)+len(locLinks) > 0, HasLocLinks: len(locLinks) > 0, LocLinks: locLinks, HasExtLinks: len(extLinks) > 0, ExtLinks: extLinks, ExtNewWindow: htmlAttrNewWindow(len(extLinks) > 0), - Matrix: matrix, + Matrix: infoAPIMatrix(zid), + Endnotes: endnotes, }) } } func splitLocExtLinks(links []*ast.Reference) (locLinks, extLinks []string) { @@ -173,5 +159,27 @@ locLinks = append(locLinks, ref.String()) } } return locLinks, extLinks } + +func infoAPIMatrix(zid id.Zid) []matrixLine { + formats := encoder.GetFormats() + defFormat := encoder.GetDefaultFormat() + parts := []string{"zettel", "meta", "content"} + matrix := make([]matrixLine, 0, len(parts)) + u := adapter.NewURLBuilder('z').SetZid(zid) + for _, part := range parts { + row := make([]matrixElement, 0, len(formats)+1) + row = append(row, matrixElement{part, false, ""}) + for _, format := range formats { + u.AppendQuery("_part", part) + if format != defFormat { + u.AppendQuery("_format", format) + } + row = append(row, matrixElement{format, true, u.String()}) + u.ClearQuery() + } + matrix = append(matrix, matrixLine{row}) + } + return matrix +} Index: web/adapter/webui/get_zettel.go ================================================================== --- web/adapter/webui/get_zettel.go +++ web/adapter/webui/get_zettel.go @@ -6,11 +6,11 @@ // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "bytes" "net/http" @@ -19,10 +19,12 @@ "zettelstore.de/z/ast" "zettelstore.de/z/config/runtime" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" "zettelstore.de/z/encoder" + "zettelstore.de/z/encoder/encfun" + "zettelstore.de/z/place" "zettelstore.de/z/usecase" "zettelstore.de/z/web/adapter" "zettelstore.de/z/web/session" ) @@ -30,76 +32,63 @@ func MakeGetHTMLZettelHandler( te *TemplateEngine, parseZettel usecase.ParseZettel, getMeta usecase.GetMeta) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() zid, err := id.Parse(r.URL.Path[1:]) if err != nil { - http.NotFound(w, r) + te.reportError(ctx, w, place.ErrNotFound) return } - ctx := r.Context() syntax := r.URL.Query().Get("syntax") zn, err := parseZettel.Run(ctx, zid, syntax) if err != nil { - adapter.ReportUsecaseError(w, err) - return - } - - metaHeader, err := formatMeta( - zn.InhMeta, - "html", - &encoder.StringsOption{ - Key: "no-meta", - Value: []string{meta.KeyTitle, meta.KeyLang}, - }, - ) - if err != nil { - adapter.InternalServerError(w, "Format meta", err) - return - } - langOption := encoder.StringOption{Key: "lang", Value: runtime.GetLang(zn.InhMeta)} - htmlTitle, err := adapter.FormatInlines(zn.Title, "html", &langOption) - if err != nil { - adapter.InternalServerError(w, "Format HTML inlines", err) - return - } - textTitle, err := adapter.FormatInlines(zn.Title, "text", &langOption) - if err != nil { - adapter.InternalServerError(w, "Format text inlines", err) - return - } - newWindow := true - htmlContent, err := formatBlocks( - zn.Ast, - "html", - &langOption, - &encoder.StringOption{ - Key: meta.KeyMarkerExternal, - Value: runtime.GetMarkerExternal()}, - &encoder.BoolOption{Key: "newwindow", Value: newWindow}, - &encoder.AdaptLinkOption{ - Adapter: adapter.MakeLinkAdapter(ctx, 'h', getMeta, "", ""), - }, - &encoder.AdaptImageOption{Adapter: adapter.MakeImageAdapter()}, - ) - if err != nil { - adapter.InternalServerError(w, "Format blocks", err) - return - } - user := session.GetUser(ctx) - roleText := zn.Zettel.Meta.GetDefault(meta.KeyRole, "*") - tags := buildTagInfos(zn.Zettel.Meta) - getTitle := makeGetTitle(ctx, getMeta, &langOption) - extURL, hasExtURL := zn.Zettel.Meta.Get(meta.KeyURL) - backLinks := formatBackLinks(zn.InhMeta, getTitle) - var base baseData - te.makeBaseData(ctx, langOption.Value, textTitle, user, &base) - base.MetaHeader = metaHeader - canCopy := base.CanCreate && !zn.Zettel.Content.IsBinary() - te.renderTemplate(ctx, w, id.DetailTemplateZid, &base, struct { + te.reportError(ctx, w, err) + return + } + + lang := runtime.GetLang(zn.InhMeta) + envHTML := encoder.Environment{ + LinkAdapter: adapter.MakeLinkAdapter(ctx, 'h', getMeta, "", ""), + ImageAdapter: adapter.MakeImageAdapter(), + CiteAdapter: nil, + Lang: lang, + Xhtml: false, + MarkerExternal: runtime.GetMarkerExternal(), + NewWindow: true, + IgnoreMeta: map[string]bool{meta.KeyTitle: true, meta.KeyLang: true}, + } + metaHeader, err := formatMeta(zn.InhMeta, "html", &envHTML) + if err != nil { + te.reportError(ctx, w, err) + return + } + htmlTitle, err := adapter.FormatInlines( + encfun.MetaAsInlineSlice(zn.InhMeta, meta.KeyTitle), "html", &envHTML) + if err != nil { + te.reportError(ctx, w, err) + return + } + htmlContent, err := formatBlocks(zn.Ast, "html", &envHTML) + if err != nil { + te.reportError(ctx, w, err) + return + } + textTitle := encfun.MetaAsText(zn.InhMeta, meta.KeyTitle) + user := session.GetUser(ctx) + roleText := zn.Meta.GetDefault(meta.KeyRole, "*") + tags := buildTagInfos(zn.Meta) + getTitle := makeGetTitle(ctx, getMeta, &encoder.Environment{Lang: lang}) + extURL, hasExtURL := zn.Meta.Get(meta.KeyURL) + backLinks := formatBackLinks(zn.InhMeta, getTitle) + var base baseData + te.makeBaseData(ctx, lang, textTitle, user, &base) + base.MetaHeader = metaHeader + canCopy := base.CanCreate && !zn.Content.IsBinary() + te.renderTemplate(ctx, w, id.ZettelTemplateZid, &base, struct { HTMLTitle string CanWrite bool EditURL string Zid string InfoURL string @@ -119,37 +108,36 @@ Content string HasBackLinks bool BackLinks []simpleLink }{ HTMLTitle: htmlTitle, - CanWrite: te.canWrite(ctx, user, zn.Zettel), + CanWrite: te.canWrite(ctx, user, zn.Meta, zn.Content), EditURL: adapter.NewURLBuilder('e').SetZid(zid).String(), Zid: zid.String(), InfoURL: adapter.NewURLBuilder('i').SetZid(zid).String(), RoleText: roleText, RoleURL: adapter.NewURLBuilder('h').AppendQuery("role", roleText).String(), HasTags: len(tags) > 0, Tags: tags, CanCopy: canCopy, CopyURL: adapter.NewURLBuilder('c').SetZid(zid).String(), - CanFolge: base.CanCreate && !zn.Zettel.Content.IsBinary(), + CanFolge: base.CanCreate && !zn.Content.IsBinary(), FolgeURL: adapter.NewURLBuilder('f').SetZid(zid).String(), FolgeRefs: formatMetaKey(zn.InhMeta, meta.KeyFolge, getTitle), PrecursorRefs: formatMetaKey(zn.InhMeta, meta.KeyPrecursor, getTitle), ExtURL: extURL, HasExtURL: hasExtURL, - ExtNewWindow: htmlAttrNewWindow(newWindow && hasExtURL), + ExtNewWindow: htmlAttrNewWindow(envHTML.NewWindow && hasExtURL), Content: htmlContent, HasBackLinks: len(backLinks) > 0, BackLinks: backLinks, }) } } -func formatBlocks( - bs ast.BlockSlice, format string, options ...encoder.Option) (string, error) { - enc := encoder.Create(format, options...) +func formatBlocks(bs ast.BlockSlice, format string, env *encoder.Environment) (string, error) { + enc := encoder.Create(format, env) if enc == nil { return "", adapter.ErrNoSuchFormat } var content strings.Builder @@ -158,12 +146,12 @@ return "", err } return content.String(), nil } -func formatMeta(m *meta.Meta, format string, options ...encoder.Option) (string, error) { - enc := encoder.Create(format, options...) +func formatMeta(m *meta.Meta, format string, env *encoder.Environment) (string, error) { + enc := encoder.Create(format, env) if enc == nil { return "", adapter.ErrNoSuchFormat } var content strings.Builder @@ -178,11 +166,11 @@ var tagInfos []simpleLink if tags, ok := m.GetList(meta.KeyTags); ok { ub := adapter.NewURLBuilder('h') tagInfos = make([]simpleLink, len(tags)) for i, tag := range tags { - tagInfos[i] = simpleLink{Text: tag, URL: ub.AppendQuery("tags", meta.CleanTag(tag)).String()} + tagInfos[i] = simpleLink{Text: tag, URL: ub.AppendQuery("tags", tag).String()} ub.ClearQuery() } } return tagInfos } Index: web/adapter/webui/home.go ================================================================== --- web/adapter/webui/home.go +++ web/adapter/webui/home.go @@ -6,11 +6,11 @@ // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "context" "net/http" @@ -28,34 +28,32 @@ // GetMeta retrieves just the meta data of a specific zettel. GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) } // MakeGetRootHandler creates a new HTTP handler to show the root URL. -func MakeGetRootHandler(s getRootStore) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/" { - http.NotFound(w, r) - return - } - ok := false - ctx := r.Context() - homeZid := runtime.GetHomeZettel() - if homeZid != id.DefaultHomeZid && homeZid.IsValid() { - if _, err := s.GetMeta(ctx, homeZid); err != nil { - homeZid = id.DefaultHomeZid - } else { - ok = true - } - } - if !ok { - if _, err := s.GetMeta(ctx, homeZid); err != nil { - if place.IsErrNotAllowed(err) && startup.WithAuth() && session.GetUser(ctx) == nil { - http.Redirect(w, r, adapter.NewURLBuilder('a').String(), http.StatusFound) - return - } - http.Redirect(w, r, adapter.NewURLBuilder('h').String(), http.StatusFound) - return - } - } - http.Redirect(w, r, adapter.NewURLBuilder('h').SetZid(homeZid).String(), http.StatusFound) +func MakeGetRootHandler(te *TemplateEngine, s getRootStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + if r.URL.Path != "/" { + te.reportError(ctx, w, place.ErrNotFound) + return + } + homeZid := runtime.GetHomeZettel() + if homeZid != id.DefaultHomeZid { + if _, err := s.GetMeta(ctx, homeZid); err == nil { + redirectFound(w, r, adapter.NewURLBuilder('h').SetZid(homeZid)) + return + } + homeZid = id.DefaultHomeZid + } + _, err := s.GetMeta(ctx, homeZid) + if err == nil { + redirectFound(w, r, adapter.NewURLBuilder('h').SetZid(homeZid)) + return + } + if place.IsErrNotAllowed(err) && startup.WithAuth() && session.GetUser(ctx) == nil { + redirectFound(w, r, adapter.NewURLBuilder('a')) + return + } + redirectFound(w, r, adapter.NewURLBuilder('h')) } } Index: web/adapter/webui/htmlmeta.go ================================================================== --- web/adapter/webui/htmlmeta.go +++ web/adapter/webui/htmlmeta.go @@ -6,11 +6,11 @@ // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "context" "fmt" @@ -29,11 +29,11 @@ "zettelstore.de/z/web/adapter" ) var space = []byte{' '} -func writeHTMLMetaValue(w io.Writer, m *meta.Meta, key string, getTitle getTitleFunc, option encoder.Option) { +func writeHTMLMetaValue(w io.Writer, m *meta.Meta, key string, getTitle getTitleFunc, env *encoder.Environment) { switch kt := m.Type(key); kt { case meta.TypeBool: writeHTMLBool(w, key, m.GetBool(key)) case meta.TypeCredential: writeCredential(w, m.GetDefault(key, "???c")) @@ -64,11 +64,11 @@ case meta.TypeWordSet: if l, ok := m.GetList(key); ok { writeWordSet(w, key, l) } case meta.TypeZettelmarkup: - writeZettelmarkup(w, m.GetDefault(key, "???z"), option) + writeZettelmarkup(w, m.GetDefault(key, "???z"), env) case meta.TypeUnknown: writeUnknown(w, m.GetDefault(key, "???u")) default: strfun.HTMLEscape(w, m.GetDefault(key, "???w"), false) fmt.Fprintf(w, " (Unhandled type: %v, key: %v)", kt, key) @@ -142,11 +142,11 @@ func writeTagSet(w io.Writer, key string, tags []string) { for i, tag := range tags { if i > 0 { w.Write(space) } - writeLink(w, key, meta.CleanTag(tag), tag) + writeLink(w, key, tag, tag) } } func writeTimestamp(w io.Writer, ts time.Time) { io.WriteString(w, ts.Format("2006-01-02 15:04:05")) @@ -173,13 +173,12 @@ w.Write(space) } writeWord(w, key, word) } } -func writeZettelmarkup(w io.Writer, val string, option encoder.Option) { - astTitle := parser.ParseTitle(val) - title, err := adapter.FormatInlines(astTitle, "html", option) +func writeZettelmarkup(w io.Writer, val string, env *encoder.Environment) { + title, err := adapter.FormatInlines(parser.ParseMetadata(val), "html", env) if err != nil { strfun.HTMLEscape(w, val, false) return } io.WriteString(w, title) @@ -193,22 +192,22 @@ io.WriteString(w, "") } type getTitleFunc func(id.Zid, string) (string, int) -func makeGetTitle(ctx context.Context, getMeta usecase.GetMeta, langOption encoder.Option) getTitleFunc { +func makeGetTitle(ctx context.Context, getMeta usecase.GetMeta, env *encoder.Environment) getTitleFunc { return func(zid id.Zid, format string) (string, int) { m, err := getMeta.Run(index.NoEnrichContext(ctx), zid) if err != nil { if place.IsErrNotAllowed(err) { return "", -1 } return "", 0 } - astTitle := parser.ParseTitle(m.GetDefault(meta.KeyTitle, "")) - title, err := adapter.FormatInlines(astTitle, format, langOption) + astTitle := parser.ParseMetadata(m.GetDefault(meta.KeyTitle, "")) + title, err := adapter.FormatInlines(astTitle, format, env) if err == nil { return title, 1 } return "", 1 } } Index: web/adapter/webui/lists.go ================================================================== --- web/adapter/webui/lists.go +++ web/adapter/webui/lists.go @@ -6,11 +6,11 @@ // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "context" "net/http" @@ -24,16 +24,18 @@ "zettelstore.de/z/domain/meta" "zettelstore.de/z/encoder" "zettelstore.de/z/index" "zettelstore.de/z/parser" "zettelstore.de/z/place" + "zettelstore.de/z/search" "zettelstore.de/z/usecase" "zettelstore.de/z/web/adapter" "zettelstore.de/z/web/session" ) -// MakeListHTMLMetaHandler creates a HTTP handler for rendering the list of zettel as HTML. +// MakeListHTMLMetaHandler creates a HTTP handler for rendering the list of +// zettel as HTML. func MakeListHTMLMetaHandler( te *TemplateEngine, listMeta usecase.ListMeta, listRole usecase.ListRole, listTags usecase.ListTags, @@ -52,20 +54,19 @@ } func renderWebUIZettelList( w http.ResponseWriter, r *http.Request, te *TemplateEngine, listMeta usecase.ListMeta) { query := r.URL.Query() - filter, sorter := adapter.GetFilterSorter(query, false) + s := adapter.GetSearch(query, false) ctx := r.Context() - title := listTitleFilterSorter("Filter", filter, sorter) + title := listTitleSearch("Filter", s) renderWebUIMetaList( - ctx, w, te, title, sorter, - func(sorter *place.Sorter) ([]*meta.Meta, error) { - if filter == nil && (sorter == nil || sorter.Order == "") { + ctx, w, te, title, s, func(s *search.Search) ([]*meta.Meta, error) { + if !s.HasComputedMetaKey() { ctx = index.NoEnrichContext(ctx) } - return listMeta.Run(ctx, filter, sorter) + return listMeta.Run(ctx, s) }, func(offset int) string { return newPageURL('h', query, offset, "_offset", "_limit") }) } @@ -128,11 +129,11 @@ ) { ctx := r.Context() iMinCount, _ := strconv.Atoi(r.URL.Query().Get("min")) tagData, err := listTags.Run(ctx, iMinCount) if err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } user := session.GetUser(ctx) tagsList := make([]tagInfo, 0, len(tagData)) @@ -165,46 +166,47 @@ var base baseData te.makeBaseData(ctx, runtime.GetDefaultLang(), runtime.GetSiteName(), user, &base) minCounts := make([]countInfo, 0, len(countList)) for _, c := range countList { sCount := strconv.Itoa(c) - minCounts = append(minCounts, countInfo{sCount, base.ListTagsURL + "?min=" + sCount}) + minCounts = append(minCounts, countInfo{sCount, base.ListTagsURL + "&min=" + sCount}) } te.renderTemplate(ctx, w, id.TagsTemplateZid, &base, struct { - MinCounts []countInfo - Tags []tagInfo + ListTagsURL string + MinCounts []countInfo + Tags []tagInfo }{ - MinCounts: minCounts, - Tags: tagsList, + ListTagsURL: base.ListTagsURL, + MinCounts: minCounts, + Tags: tagsList, }) } // MakeSearchHandler creates a new HTTP handler for the use case "search". func MakeSearchHandler( te *TemplateEngine, - search usecase.Search, + ucSearch usecase.Search, getMeta usecase.GetMeta, getZettel usecase.GetZettel, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() - filter, sorter := adapter.GetFilterSorter(query, true) - if filter == nil || len(filter.Expr) == 0 { - http.Redirect(w, r, adapter.NewURLBuilder('h').String(), http.StatusFound) + s := adapter.GetSearch(query, true) + if s == nil { + redirectFound(w, r, adapter.NewURLBuilder('h')) return } ctx := r.Context() - title := listTitleFilterSorter("Search", filter, sorter) + title := listTitleSearch("Search", s) renderWebUIMetaList( - ctx, w, te, title, sorter, - func(sorter *place.Sorter) ([]*meta.Meta, error) { - if filter == nil && (sorter == nil || sorter.Order == "") { + ctx, w, te, title, s, func(s *search.Search) ([]*meta.Meta, error) { + if !s.HasComputedMetaKey() { ctx = index.NoEnrichContext(ctx) } - return search.Run(ctx, filter, sorter) + return ucSearch.Run(ctx, s) }, func(offset int) string { return newPageURL('f', query, offset, "offset", "limit") }) } @@ -211,29 +213,23 @@ } // MakeZettelContextHandler creates a new HTTP handler for the use case "zettel context". func MakeZettelContextHandler(te *TemplateEngine, getContext usecase.ZettelContext) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() zid, err := id.Parse(r.URL.Path[1:]) if err != nil { - http.NotFound(w, r) + te.reportError(ctx, w, place.ErrNotFound) return } q := r.URL.Query() dir := usecase.ParseZCDirection(q.Get("dir")) - depth, ok := adapter.GetInteger(q, "depth") - if !ok || depth < 0 { - depth = 5 - } - limit, ok := adapter.GetInteger(q, "limit") - if !ok || limit < 0 { - limit = 200 - } - ctx := r.Context() + depth := getIntParameter(q, "depth", 5) + limit := getIntParameter(q, "limit", 200) metaList, err := getContext.Run(ctx, zid, dir, depth, limit) if err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } metaLinks, err := buildHTMLMetaList(metaList) if err != nil { adapter.InternalServerError(w, "Build HTML meta list", err) @@ -271,54 +267,61 @@ Start: metaLinks[0], Metas: metaLinks[1:], }) } } + +func getIntParameter(q url.Values, key string, minValue int) int { + val, ok := adapter.GetInteger(q, key) + if !ok || val < 0 { + return minValue + } + return val +} func renderWebUIMetaList( ctx context.Context, w http.ResponseWriter, te *TemplateEngine, title string, - sorter *place.Sorter, - ucMetaList func(sorter *place.Sorter) ([]*meta.Meta, error), + s *search.Search, + ucMetaList func(sorter *search.Search) ([]*meta.Meta, error), pageURL func(int) string) { var metaList []*meta.Meta var err error var prevURL, nextURL string if lps := runtime.GetListPageSize(); lps > 0 { - sorter = place.EnsureSorter(sorter) - if sorter.Limit < lps { - sorter.Limit = lps + 1 + if s.GetLimit() < lps { + s.SetLimit(lps + 1) } - metaList, err = ucMetaList(sorter) + metaList, err = ucMetaList(s) if err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } - if offset := sorter.Offset; offset > 0 { + if offset := s.GetOffset(); offset > 0 { offset -= lps if offset < 0 { offset = 0 } prevURL = pageURL(offset) } - if len(metaList) >= sorter.Limit { - nextURL = pageURL(sorter.Offset + lps) + if len(metaList) >= s.GetLimit() { + nextURL = pageURL(s.GetOffset() + lps) metaList = metaList[:len(metaList)-1] } } else { - metaList, err = ucMetaList(sorter) + metaList, err = ucMetaList(s) if err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } } user := session.GetUser(ctx) metas, err := buildHTMLMetaList(metaList) if err != nil { - adapter.InternalServerError(w, "Build HTML meta list", err) + te.reportError(ctx, w, err) return } var base baseData te.makeBaseData(ctx, runtime.GetDefaultLang(), runtime.GetSiteName(), user, &base) te.renderTemplate(ctx, w, id.ListTemplateZid, &base, struct { @@ -338,108 +341,23 @@ HasNext: len(nextURL) > 0, NextURL: nextURL, }) } -func listTitleFilterSorter(prefix string, filter *place.Filter, sorter *place.Sorter) string { - if filter == nil && sorter == nil { +func listTitleSearch(prefix string, s *search.Search) string { + if s == nil { return runtime.GetSiteName() } var sb strings.Builder sb.WriteString(prefix) - sb.WriteString(": ") - if filter != nil { - listTitleFilter(&sb, filter) - if sorter != nil { - sb.WriteString(" | ") - listTitleSorter(&sb, sorter) - } - } else if sorter != nil { - listTitleSorter(&sb, sorter) + if s != nil { + sb.WriteString(": ") + s.Print(&sb) } return sb.String() } -func listTitleFilter(sb *strings.Builder, filter *place.Filter) { - if filter.Negate { - sb.WriteString("NOT (") - } - names := make([]string, 0, len(filter.Expr)) - for name := range filter.Expr { - names = append(names, name) - } - sort.Strings(names) - for i, name := range names { - if i > 0 { - sb.WriteString(" AND ") - } - if name == "" { - sb.WriteString("ANY") - } else { - sb.WriteString(name) - } - sb.WriteString(" MATCH ") - writeFilterExprValues(sb, filter.Expr[name]) - } - if filter.Negate { - sb.WriteByte(')') - } -} - -func writeFilterExprValues(sb *strings.Builder, values []string) { - if len(values) == 0 { - sb.WriteString("ANY") - return - } - - for j, val := range values { - if j > 0 { - sb.WriteString(" AND ") - } - if val == "" { - sb.WriteString("ANY") - } else { - sb.WriteString(val) - } - } -} - -func listTitleSorter(sb *strings.Builder, sorter *place.Sorter) { - var space bool - if ord := sorter.Order; len(ord) > 0 { - switch ord { - case meta.KeyID: - // Ignore - case place.RandomOrder: - sb.WriteString("RANDOM") - space = true - default: - sb.WriteString("SORT ") - sb.WriteString(ord) - if sorter.Descending { - sb.WriteString(" DESC") - } - space = true - } - } - if off := sorter.Offset; off > 0 { - if space { - sb.WriteByte(' ') - } - sb.WriteString("OFFSET ") - sb.WriteString(strconv.Itoa(off)) - space = true - } - if lim := sorter.Limit; lim > 0 { - if space { - sb.WriteByte(' ') - } - sb.WriteString("LIMIT ") - sb.WriteString(strconv.Itoa(lim)) - } -} - func newPageURL(key byte, query url.Values, offset int, offsetKey, limitKey string) string { urlBuilder := adapter.NewURLBuilder(key) for key, values := range query { if key != offsetKey && key != limitKey { for _, val := range values { @@ -454,21 +372,21 @@ } // buildHTMLMetaList builds a zettel list based on a meta list for HTML rendering. func buildHTMLMetaList(metaList []*meta.Meta) ([]simpleLink, error) { defaultLang := runtime.GetDefaultLang() - langOption := encoder.StringOption{Key: "lang", Value: ""} metas := make([]simpleLink, 0, len(metaList)) for _, m := range metaList { - if lang, ok := m.Get(meta.KeyLang); ok { - langOption.Value = lang + var lang string + if val, ok := m.Get(meta.KeyLang); ok { + lang = val } else { - langOption.Value = defaultLang + lang = defaultLang } title, _ := m.Get(meta.KeyTitle) - htmlTitle, err := adapter.FormatInlines( - parser.ParseTitle(title), "html", &langOption) + env := encoder.Environment{Lang: lang, Interactive: true} + htmlTitle, err := adapter.FormatInlines(parser.ParseMetadata(title), "html", &env) if err != nil { return nil, err } metas = append(metas, simpleLink{ Text: htmlTitle, Index: web/adapter/webui/login.go ================================================================== --- web/adapter/webui/login.go +++ web/adapter/webui/login.go @@ -1,16 +1,16 @@ //----------------------------------------------------------------------------- -// Copyright (c) 2020 Detlef Stern +// Copyright (c) 2020-2021 Detlef Stern // // This file is part of zettelstore. // // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "context" "fmt" @@ -47,11 +47,11 @@ // MakePostLoginHandlerHTML creates a new HTTP handler to authenticate the given user. func MakePostLoginHandlerHTML(te *TemplateEngine, auth usecase.Authenticate) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if !startup.WithAuth() { - http.Redirect(w, r, adapter.NewURLBuilder('/').String(), http.StatusFound) + redirectFound(w, r, adapter.NewURLBuilder('/')) return } htmlDur, _ := startup.TokenLifetime() authenticateViaHTML(te, auth, w, r, htmlDur) } @@ -62,37 +62,38 @@ auth usecase.Authenticate, w http.ResponseWriter, r *http.Request, authDuration time.Duration, ) { + ctx := r.Context() ident, cred, ok := adapter.GetCredentialsViaForm(r) if !ok { - adapter.BadRequest(w, "Unable to read login form") + te.reportError(ctx, w, adapter.NewErrBadRequest("Unable to read login form")) return } - ctx := r.Context() token, err := auth.Run(ctx, ident, cred, authDuration, token.KindHTML) if err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } if token == nil { renderLoginForm(session.ClearToken(ctx, w), w, te, true) return } session.SetToken(w, token, authDuration) - http.Redirect(w, r, adapter.NewURLBuilder('/').String(), http.StatusFound) + redirectFound(w, r, adapter.NewURLBuilder('/')) } // MakeGetLogoutHandler creates a new HTTP handler to log out the current user -func MakeGetLogoutHandler() http.HandlerFunc { +func MakeGetLogoutHandler(te *TemplateEngine) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if format := adapter.GetFormat(r, r.URL.Query(), "html"); format != "html" { - adapter.BadRequest(w, fmt.Sprintf("Logout not possible in format %q", format)) + te.reportError(r.Context(), w, adapter.NewErrBadRequest( + fmt.Sprintf("Logout not possible in format %q", format))) return } session.ClearToken(r.Context(), w) - http.Redirect(w, r, adapter.NewURLBuilder('/').String(), http.StatusFound) + redirectFound(w, r, adapter.NewURLBuilder('/')) } } Index: web/adapter/webui/rename_zettel.go ================================================================== --- web/adapter/webui/rename_zettel.go +++ web/adapter/webui/rename_zettel.go @@ -6,11 +6,11 @@ // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "fmt" "net/http" @@ -17,10 +17,11 @@ "strings" "zettelstore.de/z/config/runtime" "zettelstore.de/z/domain/id" "zettelstore.de/z/domain/meta" + "zettelstore.de/z/place" "zettelstore.de/z/usecase" "zettelstore.de/z/web/adapter" "zettelstore.de/z/web/session" ) @@ -27,25 +28,26 @@ // MakeGetRenameZettelHandler creates a new HTTP handler to display the // HTML rename view of a zettel. func MakeGetRenameZettelHandler( te *TemplateEngine, getMeta usecase.GetMeta) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() zid, err := id.Parse(r.URL.Path[1:]) if err != nil { - http.NotFound(w, r) + te.reportError(ctx, w, place.ErrNotFound) return } - ctx := r.Context() m, err := getMeta.Run(ctx, zid) if err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } if format := adapter.GetFormat(r, r.URL.Query(), "html"); format != "html" { - adapter.BadRequest(w, fmt.Sprintf("Rename zettel %q not possible in format %q", zid.String(), format)) + te.reportError(ctx, w, adapter.NewErrBadRequest( + fmt.Sprintf("Rename zettel %q not possible in format %q", zid.String(), format))) return } user := session.GetUser(ctx) var base baseData @@ -59,34 +61,36 @@ }) } } // MakePostRenameZettelHandler creates a new HTTP handler to rename an existing zettel. -func MakePostRenameZettelHandler(renameZettel usecase.RenameZettel) http.HandlerFunc { +func MakePostRenameZettelHandler(te *TemplateEngine, renameZettel usecase.RenameZettel) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() curZid, err := id.Parse(r.URL.Path[1:]) if err != nil { - http.NotFound(w, r) + te.reportError(ctx, w, place.ErrNotFound) return } + if err = r.ParseForm(); err != nil { - adapter.BadRequest(w, "Unable to read rename zettel form") + te.reportError(ctx, w, adapter.NewErrBadRequest("Unable to read rename zettel form")) return } if formCurZid, err1 := id.Parse( r.PostFormValue("curzid")); err1 != nil || formCurZid != curZid { - adapter.BadRequest(w, "Invalid value for current zettel id in form") + te.reportError(ctx, w, adapter.NewErrBadRequest("Invalid value for current zettel id in form")) return } newZid, err := id.Parse(strings.TrimSpace(r.PostFormValue("newzid"))) if err != nil { - adapter.BadRequest(w, fmt.Sprintf("Invalid new zettel id %q", newZid.String())) + te.reportError(ctx, w, adapter.NewErrBadRequest(fmt.Sprintf("Invalid new zettel id %q", newZid))) return } if err := renameZettel.Run(r.Context(), curZid, newZid); err != nil { - adapter.ReportUsecaseError(w, err) + te.reportError(ctx, w, err) return } - http.Redirect(w, r, adapter.NewURLBuilder('h').SetZid(newZid).String(), http.StatusFound) + redirectFound(w, r, adapter.NewURLBuilder('h').SetZid(newZid)) } } ADDED web/adapter/webui/response.go Index: web/adapter/webui/response.go ================================================================== --- /dev/null +++ web/adapter/webui/response.go @@ -0,0 +1,22 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2021 Detlef Stern +// +// This file is part of zettelstore. +// +// Zettelstore is licensed under the latest version of the EUPL (European Union +// Public License). Please see file LICENSE.txt for your rights and obligations +// under this license. +//----------------------------------------------------------------------------- + +// Package webui provides web-UI handlers for web requests. +package webui + +import ( + "net/http" + + "zettelstore.de/z/web/adapter" +) + +func redirectFound(w http.ResponseWriter, r *http.Request, ub *adapter.URLBuilder) { + http.Redirect(w, r, ub.String(), http.StatusFound) +} Index: web/adapter/webui/template.go ================================================================== --- web/adapter/webui/template.go +++ web/adapter/webui/template.go @@ -6,16 +6,17 @@ // Zettelstore is licensed under the latest version of the EUPL (European Union // Public License). Please see file LICENSE.txt for your rights and obligations // under this license. //----------------------------------------------------------------------------- -// Package webui provides wet-UI handlers for web requests. +// Package webui provides web-UI handlers for web requests. package webui import ( "bytes" "context" + "log" "net/http" "sync" "zettelstore.de/z/auth/policy" "zettelstore.de/z/auth/token" @@ -28,20 +29,20 @@ "zettelstore.de/z/encoder" "zettelstore.de/z/index" "zettelstore.de/z/input" "zettelstore.de/z/parser" "zettelstore.de/z/place" + "zettelstore.de/z/place/change" "zettelstore.de/z/template" "zettelstore.de/z/web/adapter" "zettelstore.de/z/web/session" ) type templatePlace interface { CanCreateZettel(ctx context.Context) bool GetZettel(ctx context.Context, zid id.Zid) (domain.Zettel, error) GetMeta(ctx context.Context, zid id.Zid) (*meta.Meta, error) - SelectMeta(ctx context.Context, f *place.Filter, s *place.Sorter) ([]*meta.Meta, error) CanUpdateZettel(ctx context.Context, zettel domain.Zettel) bool AllowRenameZettel(ctx context.Context, zid id.Zid) bool CanDeleteZettel(ctx context.Context, zid id.Zid) bool } @@ -77,18 +78,18 @@ listTagsURL: adapter.NewURLBuilder('h').AppendQuery("_l", "t").String(), withAuth: startup.WithAuth(), loginURL: adapter.NewURLBuilder('a').String(), searchURL: adapter.NewURLBuilder('f').String(), } - te.observe(place.ChangeInfo{Reason: place.OnReload, Zid: id.Invalid}) + te.observe(change.Info{Reason: change.OnReload, Zid: id.Invalid}) mgr.RegisterObserver(te.observe) return te } -func (te *TemplateEngine) observe(ci place.ChangeInfo) { +func (te *TemplateEngine) observe(ci change.Info) { te.mxCache.Lock() - if ci.Reason == place.OnReload || ci.Zid == id.BaseTemplateZid { + if ci.Reason == change.OnReload || ci.Zid == id.BaseTemplateZid { te.templateCache = make(map[id.Zid]*template.Template, len(te.templateCache)) } else { delete(te.templateCache, ci.Zid) } te.mxCache.Unlock() @@ -111,13 +112,13 @@ m := meta.New(id.Invalid) return te.policy.CanCreate(user, m) && te.place.CanCreateZettel(ctx) } func (te *TemplateEngine) canWrite( - ctx context.Context, user *meta.Meta, zettel domain.Zettel) bool { - return te.policy.CanWrite(user, zettel.Meta, zettel.Meta) && - te.place.CanUpdateZettel(ctx, zettel) + ctx context.Context, user, meta *meta.Meta, content domain.Content) bool { + return te.policy.CanWrite(user, meta, meta) && + te.place.CanUpdateZettel(ctx, domain.Zettel{Meta: meta, Content: content}) } func (te *TemplateEngine) canRename(ctx context.Context, user, m *meta.Meta) bool { return te.policy.CanRename(user, m) && te.place.AllowRenameZettel(ctx, m.Zid) } @@ -240,16 +241,15 @@ } if !te.policy.CanRead(user, m) { continue } title := runtime.GetTitle(m) - langOption := encoder.StringOption{Key: "lang", Value: runtime.GetLang(m)} - astTitle := parser.ParseInlines( - input.NewInput(runtime.GetTitle(m)), meta.ValueSyntaxZmk) - menuTitle, err := adapter.FormatInlines(astTitle, "html", &langOption) + astTitle := parser.ParseInlines(input.NewInput(runtime.GetTitle(m)), meta.ValueSyntaxZmk) + env := encoder.Environment{Lang: runtime.GetLang(m)} + menuTitle, err := adapter.FormatInlines(astTitle, "html", &env) if err != nil { - menuTitle, err = adapter.FormatInlines(astTitle, "text", &langOption) + menuTitle, err = adapter.FormatInlines(astTitle, "text", nil) if err != nil { menuTitle = title } } result = append(result, simpleLink{ @@ -261,10 +261,37 @@ } func (te *TemplateEngine) renderTemplate( ctx context.Context, w http.ResponseWriter, + templateID id.Zid, + base *baseData, + data interface{}) { + te.renderTemplateStatus(ctx, w, http.StatusOK, templateID, base, data) +} + +func (te *TemplateEngine) reportError(ctx context.Context, w http.ResponseWriter, err error) { + code, text := adapter.CodeMessageFromError(err) + if code == http.StatusInternalServerError { + log.Printf("%v: %v", text, err) + } + user := session.GetUser(ctx) + var base baseData + te.makeBaseData(ctx, "en", "Error", user, &base) + te.renderTemplateStatus(ctx, w, code, id.ErrorTemplateZid, &base, struct { + ErrorTitle string + ErrorText string + }{ + ErrorTitle: http.StatusText(code), + ErrorText: text, + }) +} + +func (te *TemplateEngine) renderTemplateStatus( + ctx context.Context, + w http.ResponseWriter, + code int, templateID id.Zid, base *baseData, data interface{}) { bt, err := te.getTemplate(ctx, id.BaseTemplateZid) @@ -286,11 +313,12 @@ var content bytes.Buffer err = t.Render(&content, data) if err == nil { base.Content = content.String() w.Header().Set(adapter.ContentType, "text/html; charset=utf-8") + w.WriteHeader(code) err = bt.Render(w, base) } if err != nil { - adapter.InternalServerError(w, "Unable to render template", err) + log.Println("Unable to render template", err) } } Index: www/changes.wiki ================================================================== --- www/changes.wiki +++ www/changes.wiki @@ -1,9 +1,50 @@ Change Log + +

    Changes for Version 0.0.12 (pending)

    + -

    Changes for Version 0.0.11 (pending)

    +

    Changes for Version 0.0.11 (2021-04-05)

    + * New place schema "file" allows to read zettel from a ZIP file. + A zettel collection can now be packaged and distributed easier. + (major: server) + * Non-restricted search is a full-text search. The search string will + be normalized according to Unicode NFKD. Every character that is not a letter + or a number will be ignored for the search. It is sufficient if the words + to be searched are part of words inside a zettel, both content and metadata. + (major: api, webui) + * A zettel can be excluded from being indexed (and excluded from being found + in a search) if it contains the metadata no-index: true. + (minor: api, webui) + * Menu bar is shown when displaying error messages. + (minor: webui) + * When filtering a list of zettel, it can be specified that a given value should + not match. Previously, only the whole filter expression could be + negated (which is still possible). + (minor: api, webui) + * You can filter a zettel list by specifying that specific metadata keys must + (or must not) be present. + (minor: api, webui) + * Context of a zettel (introduced in version 0.0.10) does not take tags into account any more. + Using some tags for determining the context resulted into erratic, non-deterministic context lists. + (minor: api, webui) + * Filtering zettel depending on tag values can be both by comparing only the prefix + or the whole string. If a search value begins with '#', only zettel with the exact + tag will be returned. Otherwise a zettel will be returned if the search string + just matches the prefix of only one of its tags. + (minor: api, webui) + * Many smaller bug fixes and inprovements, to the software and to the documentation. + +A note for users of macOS: in the current release and with macOS's default values, +a zettel directory place must not contain more than approx. 250 files. There are four options +to mitigate this limitation temporarily: + # You [https://zettelstore.de/manual/h/00001004010000|re-configure] your Zettelstore to use more + than one directory place. + # You update the per-process limit of open files on macOS. + # You setup a virtualization environment to run Zettelstore on Linux or Windows. + # You wait for version 0.0.12 which addresses this issue.

    Changes for Version 0.0.10 (2021-02-26)

    * Menu item “Home” now redirects to a home zettel. Its default identifier is 000100000000. @@ -95,12 +136,12 @@ This allows these zettel to be used, even when there is another Zettelstore implementation, in another programming language. Mustache is available for approx. 48 programming languages, instead of only one for Go templates. If you modified your templates, you must adapt them to the new syntax. Otherwise the WebUI will not work. * (major) Show zettel identifier of folgezettel and precursor zettel in the header of a rendered zettel. - If a zettel has backlinks, they are shown at the botton of the page - (“Links to this zettel”). + If a zettel has real backlinks, they are shown at the botton of the page + (“Additional links to this zettel”). * (minor) All property metadata, even computed metadata is shown in the info page of a zettel. * (minor) Rendering of metadata keys title and default-title in info page changed to a full HTML output for these Zettelmarkup encoded values. * (minor) Always show the zettel identifier on the zettel detail view. Previously, the identifier was not shown if the zettel was not editable. * (minor) Do not show computed metadata in edit forms anymore. Index: www/download.wiki ================================================================== --- www/download.wiki +++ www/download.wiki @@ -7,18 +7,19 @@ * However, it is in use by the main developer since March 2020 without any damage. * It may be useful for you. It is useful for me. * Take a look at the [https://zettelstore.de/manual/|manual] to know how to start and use it.

    ZIP-ped Executables

    -Build: v0.0.10 (2021-02-26). +Build: v0.0.11 (2021-04-05). - * [/uv/zettelstore-0.0.10-linux-amd64.zip|Linux] (amd64) - * [/uv/zettelstore-0.0.10-linux-arm.zip|Linux] (arm6, e.g. Raspberry Pi) - * [/uv/zettelstore-0.0.10-windows-amd64.zip|Windows] (amd64) - * [/uv/zettelstore-0.0.10-darwin-amd64.zip|macOS] (amd64) - * [/uv/zettelstore-0.0.10-darwin-arm64.zip|macOS] (arm64, aka Apple silicon) + * [/uv/zettelstore-0.0.11-linux-amd64.zip|Linux] (amd64) + * [/uv/zettelstore-0.0.11-linux-arm.zip|Linux] (arm6, e.g. Raspberry Pi) + * [/uv/zettelstore-0.0.11-windows-amd64.zip|Windows] (amd64) + * [/uv/zettelstore-0.0.11-darwin-amd64.zip|macOS] (amd64) + * [/uv/zettelstore-0.0.11-darwin-arm64.zip|macOS] (arm64, aka Apple silicon) Unzip the appropriate file, install and execute Zettelstore according to the manual.

    Zettel for the manual

    -As a starter, you can download the zettel for the manual [/uv/manual-0.0.10.zip|here]. -Just unzip the contained files and put them into your zettel folder. +As a starter, you can download the zettel for the manual [/uv/manual-0.0.11.zip|here]. +Just unzip the contained files and put them into your zettel folder or configure +a file place to read the zettel directly from the ZIP file. Index: www/index.wiki ================================================================== --- www/index.wiki +++ www/index.wiki @@ -15,17 +15,17 @@ The software, including the manual, is licensed under the [/file?name=LICENSE.txt&ci=trunk|European Union Public License 1.2 (or later)]. [https://twitter.com/zettelstore|Stay tuned]…
    -

    Latest Release: 0.0.10 (2021-02-26)

    +

    Latest Release: 0.0.11 (2021-04-05)

    * [./download.wiki|Download] - * [./changes.wiki#0_0_10|Change Summary] - * [/timeline?p=version-0.0.10&bt=version-0.0.9&y=ci|Check-ins for version 0.0.10], - [/vdiff?to=version-0.0.10&from=version-0.0.9|content diff] - * [/timeline?df=version-0.0.10&y=ci|Check-ins derived from the 0.0.10 release], - [/vdiff?from=version-0.0.10&to=trunk|content diff] + * [./changes.wiki#0_0_11|Change Summary] + * [/timeline?p=version-0.0.11&bt=version-0.0.10&y=ci|Check-ins for version 0.0.11], + [/vdiff?to=version-0.0.11&from=version-0.0.10|content diff] + * [/timeline?df=version-0.0.11&y=ci|Check-ins derived from the 0.0.11 release], + [/vdiff?from=version-0.0.11&to=trunk|content diff] * [./plan.wiki|Limitations and planned Improvements] * [/timeline?t=release|Timeline of all past releases]

    Build instructions

    Index: www/plan.wiki ================================================================== --- www/plan.wiki +++ www/plan.wiki @@ -8,11 +8,10 @@ nor modified via the standard web interface. As a workaround, you should place your file into the directory where your zettel are stored. Make sure that the file name starts with unique 14 digits that make up the zettel identifier. * Automatic lists and transclusions are not supported in Zettelmarkup. - * The search function uses only the metadata of a zettel, not its content. * …

    Smaller limitations

    * Quoted attribute values are not yet supported in Zettelmarkup: {key="value with space"}.