1#include "crypto.hh"
2#include "globals.hh"
3#include "store-api.hh"
4#include "util.hh"
5#include "nar-info-disk-cache.hh"
6#include "thread-pool.hh"
7#include "json.hh"
8#include "derivations.hh"
9
10#include <future>
11
12
13namespace nix {
14
15
16bool Store::isInStore(const Path & path) const
17{
18    return isInDir(path, storeDir);
19}
20
21
22bool Store::isStorePath(const Path & path) const
23{
24    return isInStore(path)
25        && path.size() >= storeDir.size() + 1 + storePathHashLen
26        && path.find('/', storeDir.size() + 1) == Path::npos;
27}
28
29
30void Store::assertStorePath(const Path & path) const
31{
32    if (!isStorePath(path))
33        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
34}
35
36
37Path Store::toStorePath(const Path & path) const
38{
39    if (!isInStore(path))
40        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
41    Path::size_type slash = path.find('/', storeDir.size() + 1);
42    if (slash == Path::npos)
43        return path;
44    else
45        return Path(path, 0, slash);
46}
47
48
49Path Store::followLinksToStore(const Path & _path) const
50{
51    Path path = absPath(_path);
52    while (!isInStore(path)) {
53        if (!isLink(path)) break;
54        string target = readLink(path);
55        path = absPath(target, dirOf(path));
56    }
57    if (!isInStore(path))
58        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
59    return path;
60}
61
62
63Path Store::followLinksToStorePath(const Path & path) const
64{
65    return toStorePath(followLinksToStore(path));
66}
67
68
69string storePathToName(const Path & path)
70{
71    auto base = baseNameOf(path);
72    assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-'));
73    return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1);
74}
75
76
77string storePathToHash(const Path & path)
78{
79    auto base = baseNameOf(path);
80    assert(base.size() >= storePathHashLen);
81    return string(base, 0, storePathHashLen);
82}
83
84
85void checkStoreName(const string & name)
86{
87    string validChars = "+-._?=";
88    /* Disallow names starting with a dot for possible security
89       reasons (e.g., "." and ".."). */
90    if (string(name, 0, 1) == ".")
91        throw Error(format("illegal name: ‘%1%’") % name);
92    for (auto & i : name)
93        if (!((i >= 'A' && i <= 'Z') ||
94              (i >= 'a' && i <= 'z') ||
95              (i >= '0' && i <= '9') ||
96              validChars.find(i) != string::npos))
97        {
98            throw Error(format("invalid character ‘%1%’ in name ‘%2%’")
99                % i % name);
100        }
101}
102
103
104/* Store paths have the following form:
105
106   <store>/<h>-<name>
107
108   where
109
110   <store> = the location of the Nix store, usually /nix/store
111
112   <name> = a human readable name for the path, typically obtained
113     from the name attribute of the derivation, or the name of the
114     source file from which the store path is created.  For derivation
115     outputs other than the default "out" output, the string "-<id>"
116     is suffixed to <name>.
117
118   <h> = base-32 representation of the first 160 bits of a SHA-256
119     hash of <s>; the hash part of the store name
120
121   <s> = the string "<type>:sha256:<h2>:<store>:<name>";
122     note that it includes the location of the store as well as the
123     name to make sure that changes to either of those are reflected
124     in the hash (e.g. you won't get /nix/store/<h>-name1 and
125     /nix/store/<h>-name2 with equal hash parts).
126
127   <type> = one of:
128     "text:<r1>:<r2>:...<rN>"
129       for plain text files written to the store using
130       addTextToStore(); <r1> ... <rN> are the references of the
131       path.
132     "source"
133       for paths copied to the store using addToStore() when recursive
134       = true and hashAlgo = "sha256"
135     "output:<id>"
136       for either the outputs created by derivations, OR paths copied
137       to the store using addToStore() with recursive != true or
138       hashAlgo != "sha256" (in that case "source" is used; it's
139       silly, but it's done that way for compatibility).  <id> is the
140       name of the output (usually, "out").
141
142   <h2> = base-16 representation of a SHA-256 hash of:
143     if <type> = "text:...":
144       the string written to the resulting store path
145     if <type> = "source":
146       the serialisation of the path from which this store path is
147       copied, as returned by hashPath()
148     if <type> = "output:<id>":
149       for non-fixed derivation outputs:
150         the derivation (see hashDerivationModulo() in
151         primops.cc)
152       for paths copied by addToStore() or produced by fixed-output
153       derivations:
154         the string "fixed:out:<rec><algo>:<hash>:", where
155           <rec> = "r:" for recursive (path) hashes, or "" for flat
156             (file) hashes
157           <algo> = "md5", "sha1" or "sha256"
158           <hash> = base-16 representation of the path or flat hash of
159             the contents of the path (or expected contents of the
160             path for fixed-output derivations)
161
162   It would have been nicer to handle fixed-output derivations under
163   "source", e.g. have something like "source:<rec><algo>", but we're
164   stuck with this for now...
165
166   The main reason for this way of computing names is to prevent name
167   collisions (for security).  For instance, it shouldn't be feasible
168   to come up with a derivation whose output path collides with the
169   path for a copied source.  The former would have a <s> starting with
170   "output:out:", while the latter would have a <2> starting with
171   "source:".
172*/
173
174
175Path Store::makeStorePath(const string & type,
176    const Hash & hash, const string & name) const
177{
178    /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
179    string s = type + ":sha256:" + printHash(hash) + ":"
180        + storeDir + ":" + name;
181
182    checkStoreName(name);
183
184    return storeDir + "/"
185        + printHash32(compressHash(hashString(htSHA256, s), 20))
186        + "-" + name;
187}
188
189
190Path Store::makeOutputPath(const string & id,
191    const Hash & hash, const string & name) const
192{
193    return makeStorePath("output:" + id, hash,
194        name + (id == "out" ? "" : "-" + id));
195}
196
197
198Path Store::makeFixedOutputPath(bool recursive,
199    const Hash & hash, const string & name) const
200{
201    return hash.type == htSHA256 && recursive
202        ? makeStorePath("source", hash, name)
203        : makeStorePath("output:out", hashString(htSHA256,
204                "fixed:out:" + (recursive ? (string) "r:" : "") +
205                printHashType(hash.type) + ":" + printHash(hash) + ":"),
206            name);
207}
208
209
210Path Store::makeTextPath(const string & name, const Hash & hash,
211    const PathSet & references) const
212{
213    assert(hash.type == htSHA256);
214    /* Stuff the references (if any) into the type.  This is a bit
215       hacky, but we can't put them in `s' since that would be
216       ambiguous. */
217    string type = "text";
218    for (auto & i : references) {
219        type += ":";
220        type += i;
221    }
222    return makeStorePath(type, hash, name);
223}
224
225
226std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath,
227    bool recursive, HashType hashAlgo, PathFilter & filter) const
228{
229    Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
230    string name = baseNameOf(srcPath);
231    Path dstPath = makeFixedOutputPath(recursive, h, name);
232    return std::pair<Path, Hash>(dstPath, h);
233}
234
235
236Path Store::computeStorePathForText(const string & name, const string & s,
237    const PathSet & references) const
238{
239    return makeTextPath(name, hashString(htSHA256, s), references);
240}
241
242
243Store::Store(const Params & params)
244    : Config(params)
245    , state({(size_t) pathInfoCacheSize})
246{
247}
248
249
250std::string Store::getUri()
251{
252    return "";
253}
254
255
256bool Store::isValidPath(const Path & storePath)
257{
258    auto hashPart = storePathToHash(storePath);
259
260    {
261        auto state_(state.lock());
262        auto res = state_->pathInfoCache.get(hashPart);
263        if (res) {
264            stats.narInfoReadAverted++;
265            return *res != 0;
266        }
267    }
268
269    if (diskCache) {
270        auto res = diskCache->lookupNarInfo(getUri(), hashPart);
271        if (res.first != NarInfoDiskCache::oUnknown) {
272            stats.narInfoReadAverted++;
273            auto state_(state.lock());
274            state_->pathInfoCache.upsert(hashPart,
275                res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
276            return res.first == NarInfoDiskCache::oValid;
277        }
278    }
279
280    bool valid = isValidPathUncached(storePath);
281
282    if (diskCache && !valid)
283        // FIXME: handle valid = true case.
284        diskCache->upsertNarInfo(getUri(), hashPart, 0);
285
286    return valid;
287}
288
289
290/* Default implementation for stores that only implement
291   queryPathInfoUncached(). */
292bool Store::isValidPathUncached(const Path & path)
293{
294    try {
295        queryPathInfo(path);
296        return true;
297    } catch (InvalidPath &) {
298        return false;
299    }
300}
301
302
303ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
304{
305    std::promise<ref<ValidPathInfo>> promise;
306
307    queryPathInfo(storePath,
308        [&](ref<ValidPathInfo> info) {
309            promise.set_value(info);
310        },
311        [&](std::exception_ptr exc) {
312            promise.set_exception(exc);
313        });
314
315    return promise.get_future().get();
316}
317
318
319void Store::queryPathInfo(const Path & storePath,
320    std::function<void(ref<ValidPathInfo>)> success,
321    std::function<void(std::exception_ptr exc)> failure)
322{
323    auto hashPart = storePathToHash(storePath);
324
325    try {
326
327        {
328            auto res = state.lock()->pathInfoCache.get(hashPart);
329            if (res) {
330                stats.narInfoReadAverted++;
331                if (!*res)
332                    throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
333                return success(ref<ValidPathInfo>(*res));
334            }
335        }
336
337        if (diskCache) {
338            auto res = diskCache->lookupNarInfo(getUri(), hashPart);
339            if (res.first != NarInfoDiskCache::oUnknown) {
340                stats.narInfoReadAverted++;
341                {
342                    auto state_(state.lock());
343                    state_->pathInfoCache.upsert(hashPart,
344                        res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
345                    if (res.first == NarInfoDiskCache::oInvalid ||
346                        (res.second->path != storePath && storePathToName(storePath) != ""))
347                        throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
348                }
349                return success(ref<ValidPathInfo>(res.second));
350            }
351        }
352
353    } catch (std::exception & e) {
354        return callFailure(failure);
355    }
356
357    queryPathInfoUncached(storePath,
358        [this, storePath, hashPart, success, failure](std::shared_ptr<ValidPathInfo> info) {
359
360            if (diskCache)
361                diskCache->upsertNarInfo(getUri(), hashPart, info);
362
363            {
364                auto state_(state.lock());
365                state_->pathInfoCache.upsert(hashPart, info);
366            }
367
368            if (!info
369                || (info->path != storePath && storePathToName(storePath) != ""))
370            {
371                stats.narInfoMissing++;
372                return failure(std::make_exception_ptr(InvalidPath(format("path ‘%s’ is not valid") % storePath)));
373            }
374
375            callSuccess(success, failure, ref<ValidPathInfo>(info));
376
377        }, failure);
378}
379
380
381PathSet Store::queryValidPaths(const PathSet & paths, bool maybeSubstitute)
382{
383    struct State
384    {
385        size_t left;
386        PathSet valid;
387        std::exception_ptr exc;
388    };
389
390    Sync<State> state_(State{paths.size(), PathSet()});
391
392    std::condition_variable wakeup;
393
394    for (auto & path : paths)
395        queryPathInfo(path,
396            [path, &state_, &wakeup](ref<ValidPathInfo> info) {
397                auto state(state_.lock());
398                state->valid.insert(path);
399                assert(state->left);
400                if (!--state->left)
401                    wakeup.notify_one();
402            },
403            [path, &state_, &wakeup](std::exception_ptr exc) {
404                auto state(state_.lock());
405                try {
406                    std::rethrow_exception(exc);
407                } catch (InvalidPath &) {
408                } catch (...) {
409                    state->exc = exc;
410                }
411                assert(state->left);
412                if (!--state->left)
413                    wakeup.notify_one();
414            });
415
416    while (true) {
417        auto state(state_.lock());
418        if (!state->left) {
419            if (state->exc) std::rethrow_exception(state->exc);
420            return state->valid;
421        }
422        state.wait(wakeup);
423    }
424}
425
426
427/* Return a string accepted by decodeValidPathInfo() that
428   registers the specified paths as valid.  Note: it's the
429   responsibility of the caller to provide a closure. */
430string Store::makeValidityRegistration(const PathSet & paths,
431    bool showDerivers, bool showHash)
432{
433    string s = "";
434
435    for (auto & i : paths) {
436        s += i + "\n";
437
438        auto info = queryPathInfo(i);
439
440        if (showHash) {
441            s += printHash(info->narHash) + "\n";
442            s += (format("%1%\n") % info->narSize).str();
443        }
444
445        Path deriver = showDerivers ? info->deriver : "";
446        s += deriver + "\n";
447
448        s += (format("%1%\n") % info->references.size()).str();
449
450        for (auto & j : info->references)
451            s += j + "\n";
452    }
453
454    return s;
455}
456
457
458void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
459    bool includeImpureInfo, bool showClosureSize)
460{
461    auto jsonList = jsonOut.list();
462
463    for (auto storePath : storePaths) {
464        auto info = queryPathInfo(storePath);
465        storePath = info->path;
466
467        auto jsonPath = jsonList.object();
468        jsonPath
469            .attr("path", storePath)
470            .attr("narHash", info->narHash.to_string())
471            .attr("narSize", info->narSize);
472
473        {
474            auto jsonRefs = jsonPath.list("references");
475            for (auto & ref : info->references)
476                jsonRefs.elem(ref);
477        }
478
479        if (info->ca != "")
480            jsonPath.attr("ca", info->ca);
481
482        if (showClosureSize)
483            jsonPath.attr("closureSize", getClosureSize(storePath));
484
485        if (!includeImpureInfo) continue;
486
487        if (info->deriver != "")
488            jsonPath.attr("deriver", info->deriver);
489
490        if (info->registrationTime)
491            jsonPath.attr("registrationTime", info->registrationTime);
492
493        if (info->ultimate)
494            jsonPath.attr("ultimate", info->ultimate);
495
496        if (!info->sigs.empty()) {
497            auto jsonSigs = jsonPath.list("signatures");
498            for (auto & sig : info->sigs)
499                jsonSigs.elem(sig);
500        }
501    }
502}
503
504
505unsigned long long Store::getClosureSize(const Path & storePath)
506{
507    unsigned long long totalSize = 0;
508    PathSet closure;
509    computeFSClosure(storePath, closure, false, false);
510    for (auto & p : closure)
511        totalSize += queryPathInfo(p)->narSize;
512    return totalSize;
513}
514
515
516const Store::Stats & Store::getStats()
517{
518    {
519        auto state_(state.lock());
520        stats.pathInfoCacheSize = state_->pathInfoCache.size();
521    }
522    return stats;
523}
524
525
526void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
527    const Path & storePath, bool repair, bool dontCheckSigs)
528{
529    auto info = srcStore->queryPathInfo(storePath);
530
531    StringSink sink;
532    srcStore->narFromPath({storePath}, sink);
533
534    if (srcStore->isTrusted())
535        dontCheckSigs = true;
536
537    if (!info->narHash && dontCheckSigs) {
538        auto info2 = make_ref<ValidPathInfo>(*info);
539        info2->narHash = hashString(htSHA256, *sink.s);
540        info = info2;
541    }
542
543    dstStore->addToStore(*info, sink.s, repair, dontCheckSigs);
544}
545
546
547void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
548    const PathSet & storePaths, bool repair, bool dontCheckSigs)
549{
550    PathSet closure;
551    for (auto & path : storePaths)
552        srcStore->computeFSClosure(path, closure);
553
554    // FIXME: use copyStorePaths()
555
556    PathSet valid = dstStore->queryValidPaths(closure);
557
558    if (valid.size() == closure.size()) return;
559
560    Paths sorted = srcStore->topoSortPaths(closure);
561
562    Paths missing;
563    for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
564        if (!valid.count(*i)) missing.push_back(*i);
565
566    printMsg(lvlDebug, format("copying %1% missing paths") % missing.size());
567
568    for (auto & i : missing)
569        copyStorePath(srcStore, dstStore, i, repair, dontCheckSigs);
570}
571
572
573ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
574{
575    ValidPathInfo info;
576    getline(str, info.path);
577    if (str.eof()) { info.path = ""; return info; }
578    if (hashGiven) {
579        string s;
580        getline(str, s);
581        info.narHash = parseHash(htSHA256, s);
582        getline(str, s);
583        if (!string2Int(s, info.narSize)) throw Error("number expected");
584    }
585    getline(str, info.deriver);
586    string s; int n;
587    getline(str, s);
588    if (!string2Int(s, n)) throw Error("number expected");
589    while (n--) {
590        getline(str, s);
591        info.references.insert(s);
592    }
593    if (!str || str.eof()) throw Error("missing input");
594    return info;
595}
596
597
598string showPaths(const PathSet & paths)
599{
600    string s;
601    for (auto & i : paths) {
602        if (s.size() != 0) s += ", ";
603        s += "‘" + i + "’";
604    }
605    return s;
606}
607
608
609std::string ValidPathInfo::fingerprint() const
610{
611    if (narSize == 0 || !narHash)
612        throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known")
613            % path);
614    return
615        "1;" + path + ";"
616        + printHashType(narHash.type) + ":" + printHash32(narHash) + ";"
617        + std::to_string(narSize) + ";"
618        + concatStringsSep(",", references);
619}
620
621
622void ValidPathInfo::sign(const SecretKey & secretKey)
623{
624    sigs.insert(secretKey.signDetached(fingerprint()));
625}
626
627
628bool ValidPathInfo::isContentAddressed(const Store & store) const
629{
630    auto warn = [&]() {
631        printError(format("warning: path ‘%s’ claims to be content-addressed but isn't") % path);
632    };
633
634    if (hasPrefix(ca, "text:")) {
635        auto hash = parseHash(std::string(ca, 5));
636        if (store.makeTextPath(storePathToName(path), hash, references) == path)
637            return true;
638        else
639            warn();
640    }
641
642    else if (hasPrefix(ca, "fixed:")) {
643        bool recursive = ca.compare(6, 2, "r:") == 0;
644        auto hash = parseHash(std::string(ca, recursive ? 8 : 6));
645        if (store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
646            return true;
647        else
648            warn();
649    }
650
651    return false;
652}
653
654
655size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const
656{
657    if (isContentAddressed(store)) return maxSigs;
658
659    size_t good = 0;
660    for (auto & sig : sigs)
661        if (checkSignature(publicKeys, sig))
662            good++;
663    return good;
664}
665
666
667bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
668{
669    return verifyDetached(fingerprint(), sig, publicKeys);
670}
671
672
673Strings ValidPathInfo::shortRefs() const
674{
675    Strings refs;
676    for (auto & r : references)
677        refs.push_back(baseNameOf(r));
678    return refs;
679}
680
681
682std::string makeFixedOutputCA(bool recursive, const Hash & hash)
683{
684    return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
685}
686
687
688}
689
690
691#include "local-store.hh"
692#include "remote-store.hh"
693
694
695namespace nix {
696
697
698RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
699
700
701ref<Store> openStore(const std::string & uri_)
702{
703    auto uri(uri_);
704    Store::Params params;
705    auto q = uri.find('?');
706    if (q != std::string::npos) {
707        for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
708            auto e = s.find('=');
709            if (e != std::string::npos)
710                params[s.substr(0, e)] = s.substr(e + 1);
711        }
712        uri = uri_.substr(0, q);
713    }
714    return openStore(uri, params);
715}
716
717ref<Store> openStore(const std::string & uri, const Store::Params & params)
718{
719    for (auto fun : *RegisterStoreImplementation::implementations) {
720        auto store = fun(uri, params);
721        if (store) {
722            store->warnUnknownSettings();
723            return ref<Store>(store);
724        }
725    }
726
727    throw Error(format("don't know how to open Nix store ‘%s’") % uri);
728}
729
730
731StoreType getStoreType(const std::string & uri, const std::string & stateDir)
732{
733    if (uri == "daemon") {
734        return tDaemon;
735    } else if (uri == "local") {
736        return tLocal;
737    } else if (uri == "" || uri == "auto") {
738        if (access(stateDir.c_str(), R_OK | W_OK) == 0)
739            return tLocal;
740        else if (pathExists(settings.nixDaemonSocketFile))
741            return tDaemon;
742        else
743            return tLocal;
744    } else {
745        return tOther;
746    }
747}
748
749
750static RegisterStoreImplementation regStore([](
751    const std::string & uri, const Store::Params & params)
752    -> std::shared_ptr<Store>
753{
754    switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
755        case tDaemon:
756            return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
757        case tLocal:
758            return std::shared_ptr<Store>(std::make_shared<LocalStore>(params));
759        default:
760            return nullptr;
761    }
762});
763
764
765std::list<ref<Store>> getDefaultSubstituters()
766{
767    struct State {
768        bool done = false;
769        std::list<ref<Store>> stores;
770    };
771    static Sync<State> state_;
772
773    auto state(state_.lock());
774
775    if (state->done) return state->stores;
776
777    StringSet done;
778
779    auto addStore = [&](const std::string & uri) {
780        if (done.count(uri)) return;
781        done.insert(uri);
782        state->stores.push_back(openStore(uri));
783    };
784
785    for (auto uri : settings.substituters.get())
786        addStore(uri);
787
788    for (auto uri : settings.extraSubstituters.get())
789        addStore(uri);
790
791    state->done = true;
792
793    return state->stores;
794}
795
796
797void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute)
798{
799    PathSet valid = to->queryValidPaths(storePaths, substitute);
800
801    PathSet missing;
802    for (auto & path : storePaths)
803        if (!valid.count(path)) missing.insert(path);
804
805    std::string copiedLabel = "copied";
806
807    logger->setExpected(copiedLabel, missing.size());
808
809    ThreadPool pool;
810
811    processGraph<Path>(pool,
812        PathSet(missing.begin(), missing.end()),
813
814        [&](const Path & storePath) {
815            if (to->isValidPath(storePath)) return PathSet();
816            return from->queryPathInfo(storePath)->references;
817        },
818
819        [&](const Path & storePath) {
820            checkInterrupt();
821
822            if (!to->isValidPath(storePath)) {
823                Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath);
824
825                copyStorePath(from, to, storePath);
826
827                logger->incProgress(copiedLabel);
828            } else
829                logger->incExpected(copiedLabel, -1);
830        });
831
832    pool.process();
833}
834
835
836}
837