1#include "crypto.hh"
2#include "globals.hh"
3#include "store-api.hh"
4#include "util.hh"
5#include "nar-info-disk-cache.hh"
6#include "thread-pool.hh"
7#include "json.hh"
8#include "derivations.hh"
9
10#include <future>
11
12
13namespace nix {
14
15
16bool Store::isInStore(const Path & path) const
17{
18    return isInDir(path, storeDir);
19}
20
21
22bool Store::isStorePath(const Path & path) const
23{
24    return isInStore(path)
25        && path.size() >= storeDir.size() + 1 + storePathHashLen
26        && path.find('/', storeDir.size() + 1) == Path::npos;
27}
28
29
30void Store::assertStorePath(const Path & path) const
31{
32    if (!isStorePath(path))
33        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
34}
35
36
37Path Store::toStorePath(const Path & path) const
38{
39    if (!isInStore(path))
40        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
41    Path::size_type slash = path.find('/', storeDir.size() + 1);
42    if (slash == Path::npos)
43        return path;
44    else
45        return Path(path, 0, slash);
46}
47
48
49Path Store::followLinksToStore(const Path & _path) const
50{
51    Path path = absPath(_path);
52    while (!isInStore(path)) {
53        if (!isLink(path)) break;
54        string target = readLink(path);
55        path = absPath(target, dirOf(path));
56    }
57    if (!isInStore(path))
58        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
59    return path;
60}
61
62
63Path Store::followLinksToStorePath(const Path & path) const
64{
65    return toStorePath(followLinksToStore(path));
66}
67
68
69string storePathToName(const Path & path)
70{
71    auto base = baseNameOf(path);
72    assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-'));
73    return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1);
74}
75
76
77string storePathToHash(const Path & path)
78{
79    auto base = baseNameOf(path);
80    assert(base.size() >= storePathHashLen);
81    return string(base, 0, storePathHashLen);
82}
83
84
85void checkStoreName(const string & name)
86{
87    string validChars = "+-._?=";
88    /* Disallow names starting with a dot for possible security
89       reasons (e.g., "." and ".."). */
90    if (string(name, 0, 1) == ".")
91        throw Error(format("illegal name: ‘%1%’") % name);
92    for (auto & i : name)
93        if (!((i >= 'A' && i <= 'Z') ||
94              (i >= 'a' && i <= 'z') ||
95              (i >= '0' && i <= '9') ||
96              validChars.find(i) != string::npos))
97        {
98            throw Error(format("invalid character ‘%1%’ in name ‘%2%’")
99                % i % name);
100        }
101}
102
103
104/* Store paths have the following form:
105
106   <store>/<h>-<name>
107
108   where
109
110   <store> = the location of the Nix store, usually /nix/store
111
112   <name> = a human readable name for the path, typically obtained
113     from the name attribute of the derivation, or the name of the
114     source file from which the store path is created.  For derivation
115     outputs other than the default "out" output, the string "-<id>"
116     is suffixed to <name>.
117
118   <h> = base-32 representation of the first 160 bits of a SHA-256
119     hash of <s>; the hash part of the store name
120
121   <s> = the string "<type>:sha256:<h2>:<store>:<name>";
122     note that it includes the location of the store as well as the
123     name to make sure that changes to either of those are reflected
124     in the hash (e.g. you won't get /nix/store/<h>-name1 and
125     /nix/store/<h>-name2 with equal hash parts).
126
127   <type> = one of:
128     "text:<r1>:<r2>:...<rN>"
129       for plain text files written to the store using
130       addTextToStore(); <r1> ... <rN> are the references of the
131       path.
132     "source"
133       for paths copied to the store using addToStore() when recursive
134       = true and hashAlgo = "sha256"
135     "output:<id>"
136       for either the outputs created by derivations, OR paths copied
137       to the store using addToStore() with recursive != true or
138       hashAlgo != "sha256" (in that case "source" is used; it's
139       silly, but it's done that way for compatibility).  <id> is the
140       name of the output (usually, "out").
141
142   <h2> = base-16 representation of a SHA-256 hash of:
143     if <type> = "text:...":
144       the string written to the resulting store path
145     if <type> = "source":
146       the serialisation of the path from which this store path is
147       copied, as returned by hashPath()
148     if <type> = "output:<id>":
149       for non-fixed derivation outputs:
150         the derivation (see hashDerivationModulo() in
151         primops.cc)
152       for paths copied by addToStore() or produced by fixed-output
153       derivations:
154         the string "fixed:out:<rec><algo>:<hash>:", where
155           <rec> = "r:" for recursive (path) hashes, or "" for flat
156             (file) hashes
157           <algo> = "md5", "sha1" or "sha256"
158           <hash> = base-16 representation of the path or flat hash of
159             the contents of the path (or expected contents of the
160             path for fixed-output derivations)
161
162   It would have been nicer to handle fixed-output derivations under
163   "source", e.g. have something like "source:<rec><algo>", but we're
164   stuck with this for now...
165
166   The main reason for this way of computing names is to prevent name
167   collisions (for security).  For instance, it shouldn't be feasible
168   to come up with a derivation whose output path collides with the
169   path for a copied source.  The former would have a <s> starting with
170   "output:out:", while the latter would have a <2> starting with
171   "source:".
172*/
173
174
175Path Store::makeStorePath(const string & type,
176    const Hash & hash, const string & name) const
177{
178    /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
179    string s = type + ":sha256:" + printHash(hash) + ":"
180        + storeDir + ":" + name;
181
182    checkStoreName(name);
183
184    return storeDir + "/"
185        + printHash32(compressHash(hashString(htSHA256, s), 20))
186        + "-" + name;
187}
188
189
190Path Store::makeOutputPath(const string & id,
191    const Hash & hash, const string & name) const
192{
193    return makeStorePath("output:" + id, hash,
194        name + (id == "out" ? "" : "-" + id));
195}
196
197
198Path Store::makeFixedOutputPath(bool recursive,
199    const Hash & hash, const string & name) const
200{
201    return hash.type == htSHA256 && recursive
202        ? makeStorePath("source", hash, name)
203        : makeStorePath("output:out", hashString(htSHA256,
204                "fixed:out:" + (recursive ? (string) "r:" : "") +
205                printHashType(hash.type) + ":" + printHash(hash) + ":"),
206            name);
207}
208
209
210Path Store::makeTextPath(const string & name, const Hash & hash,
211    const PathSet & references) const
212{
213    assert(hash.type == htSHA256);
214    /* Stuff the references (if any) into the type.  This is a bit
215       hacky, but we can't put them in `s' since that would be
216       ambiguous. */
217    string type = "text";
218    for (auto & i : references) {
219        type += ":";
220        type += i;
221    }
222    return makeStorePath(type, hash, name);
223}
224
225
226std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath,
227    bool recursive, HashType hashAlgo, PathFilter & filter) const
228{
229    Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
230    string name = baseNameOf(srcPath);
231    Path dstPath = makeFixedOutputPath(recursive, h, name);
232    return std::pair<Path, Hash>(dstPath, h);
233}
234
235
236Path Store::computeStorePathForText(const string & name, const string & s,
237    const PathSet & references) const
238{
239    return makeTextPath(name, hashString(htSHA256, s), references);
240}
241
242
243Store::Store(const Params & params)
244    : storeDir(get(params, "store", settings.nixStore))
245{
246}
247
248
249std::string Store::getUri()
250{
251    return "";
252}
253
254
255bool Store::isValidPath(const Path & storePath)
256{
257    auto hashPart = storePathToHash(storePath);
258
259    {
260        auto state_(state.lock());
261        auto res = state_->pathInfoCache.get(hashPart);
262        if (res) {
263            stats.narInfoReadAverted++;
264            return *res != 0;
265        }
266    }
267
268    if (diskCache) {
269        auto res = diskCache->lookupNarInfo(getUri(), hashPart);
270        if (res.first != NarInfoDiskCache::oUnknown) {
271            stats.narInfoReadAverted++;
272            auto state_(state.lock());
273            state_->pathInfoCache.upsert(hashPart,
274                res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
275            return res.first == NarInfoDiskCache::oValid;
276        }
277    }
278
279    bool valid = isValidPathUncached(storePath);
280
281    if (diskCache && !valid)
282        // FIXME: handle valid = true case.
283        diskCache->upsertNarInfo(getUri(), hashPart, 0);
284
285    return valid;
286}
287
288
289/* Default implementation for stores that only implement
290   queryPathInfoUncached(). */
291bool Store::isValidPathUncached(const Path & path)
292{
293    try {
294        queryPathInfo(path);
295        return true;
296    } catch (InvalidPath &) {
297        return false;
298    }
299}
300
301
302ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
303{
304    std::promise<ref<ValidPathInfo>> promise;
305
306    queryPathInfo(storePath,
307        [&](ref<ValidPathInfo> info) {
308            promise.set_value(info);
309        },
310        [&](std::exception_ptr exc) {
311            promise.set_exception(exc);
312        });
313
314    return promise.get_future().get();
315}
316
317
318void Store::queryPathInfo(const Path & storePath,
319    std::function<void(ref<ValidPathInfo>)> success,
320    std::function<void(std::exception_ptr exc)> failure)
321{
322    auto hashPart = storePathToHash(storePath);
323
324    try {
325
326        {
327            auto res = state.lock()->pathInfoCache.get(hashPart);
328            if (res) {
329                stats.narInfoReadAverted++;
330                if (!*res)
331                    throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
332                return success(ref<ValidPathInfo>(*res));
333            }
334        }
335
336        if (diskCache) {
337            auto res = diskCache->lookupNarInfo(getUri(), hashPart);
338            if (res.first != NarInfoDiskCache::oUnknown) {
339                stats.narInfoReadAverted++;
340                {
341                    auto state_(state.lock());
342                    state_->pathInfoCache.upsert(hashPart,
343                        res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
344                    if (res.first == NarInfoDiskCache::oInvalid ||
345                        (res.second->path != storePath && storePathToName(storePath) != ""))
346                        throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
347                }
348                return success(ref<ValidPathInfo>(res.second));
349            }
350        }
351
352    } catch (std::exception & e) {
353        return callFailure(failure);
354    }
355
356    queryPathInfoUncached(storePath,
357        [this, storePath, hashPart, success, failure](std::shared_ptr<ValidPathInfo> info) {
358
359            if (diskCache)
360                diskCache->upsertNarInfo(getUri(), hashPart, info);
361
362            {
363                auto state_(state.lock());
364                state_->pathInfoCache.upsert(hashPart, info);
365            }
366
367            if (!info
368                || (info->path != storePath && storePathToName(storePath) != ""))
369            {
370                stats.narInfoMissing++;
371                return failure(std::make_exception_ptr(InvalidPath(format("path ‘%s’ is not valid") % storePath)));
372            }
373
374            callSuccess(success, failure, ref<ValidPathInfo>(info));
375
376        }, failure);
377}
378
379
380PathSet Store::queryValidPaths(const PathSet & paths, bool maybeSubstitute)
381{
382    struct State
383    {
384        size_t left;
385        PathSet valid;
386        std::exception_ptr exc;
387    };
388
389    Sync<State> state_(State{paths.size(), PathSet()});
390
391    std::condition_variable wakeup;
392
393    for (auto & path : paths)
394        queryPathInfo(path,
395            [path, &state_, &wakeup](ref<ValidPathInfo> info) {
396                auto state(state_.lock());
397                state->valid.insert(path);
398                assert(state->left);
399                if (!--state->left)
400                    wakeup.notify_one();
401            },
402            [path, &state_, &wakeup](std::exception_ptr exc) {
403                auto state(state_.lock());
404                try {
405                    std::rethrow_exception(exc);
406                } catch (InvalidPath &) {
407                } catch (...) {
408                    state->exc = exc;
409                }
410                assert(state->left);
411                if (!--state->left)
412                    wakeup.notify_one();
413            });
414
415    while (true) {
416        auto state(state_.lock());
417        if (!state->left) {
418            if (state->exc) std::rethrow_exception(state->exc);
419            return state->valid;
420        }
421        state.wait(wakeup);
422    }
423}
424
425
426/* Return a string accepted by decodeValidPathInfo() that
427   registers the specified paths as valid.  Note: it's the
428   responsibility of the caller to provide a closure. */
429string Store::makeValidityRegistration(const PathSet & paths,
430    bool showDerivers, bool showHash)
431{
432    string s = "";
433
434    for (auto & i : paths) {
435        s += i + "\n";
436
437        auto info = queryPathInfo(i);
438
439        if (showHash) {
440            s += printHash(info->narHash) + "\n";
441            s += (format("%1%\n") % info->narSize).str();
442        }
443
444        Path deriver = showDerivers ? info->deriver : "";
445        s += deriver + "\n";
446
447        s += (format("%1%\n") % info->references.size()).str();
448
449        for (auto & j : info->references)
450            s += j + "\n";
451    }
452
453    return s;
454}
455
456
457void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
458    bool includeImpureInfo, bool showClosureSize)
459{
460    auto jsonList = jsonOut.list();
461
462    for (auto storePath : storePaths) {
463        auto info = queryPathInfo(storePath);
464        storePath = info->path;
465
466        auto jsonPath = jsonList.object();
467        jsonPath
468            .attr("path", storePath)
469            .attr("narHash", info->narHash.to_string())
470            .attr("narSize", info->narSize);
471
472        {
473            auto jsonRefs = jsonPath.list("references");
474            for (auto & ref : info->references)
475                jsonRefs.elem(ref);
476        }
477
478        if (info->ca != "")
479            jsonPath.attr("ca", info->ca);
480
481        if (showClosureSize)
482            jsonPath.attr("closureSize", getClosureSize(storePath));
483
484        if (!includeImpureInfo) continue;
485
486        if (info->deriver != "")
487            jsonPath.attr("deriver", info->deriver);
488
489        if (info->registrationTime)
490            jsonPath.attr("registrationTime", info->registrationTime);
491
492        if (info->ultimate)
493            jsonPath.attr("ultimate", info->ultimate);
494
495        if (!info->sigs.empty()) {
496            auto jsonSigs = jsonPath.list("signatures");
497            for (auto & sig : info->sigs)
498                jsonSigs.elem(sig);
499        }
500    }
501}
502
503
504unsigned long long Store::getClosureSize(const Path & storePath)
505{
506    unsigned long long totalSize = 0;
507    PathSet closure;
508    computeFSClosure(storePath, closure, false, false);
509    for (auto & p : closure)
510        totalSize += queryPathInfo(p)->narSize;
511    return totalSize;
512}
513
514
515const Store::Stats & Store::getStats()
516{
517    {
518        auto state_(state.lock());
519        stats.pathInfoCacheSize = state_->pathInfoCache.size();
520    }
521    return stats;
522}
523
524
525void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
526    const Path & storePath, bool repair, bool dontCheckSigs)
527{
528    auto info = srcStore->queryPathInfo(storePath);
529
530    StringSink sink;
531    srcStore->narFromPath({storePath}, sink);
532
533    if (srcStore->isTrusted())
534        dontCheckSigs = true;
535
536    if (!info->narHash && dontCheckSigs) {
537        auto info2 = make_ref<ValidPathInfo>(*info);
538        info2->narHash = hashString(htSHA256, *sink.s);
539        info = info2;
540    }
541
542    dstStore->addToStore(*info, sink.s, repair, dontCheckSigs);
543}
544
545
546void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
547    const PathSet & storePaths, bool repair, bool dontCheckSigs)
548{
549    PathSet closure;
550    for (auto & path : storePaths)
551        srcStore->computeFSClosure(path, closure);
552
553    // FIXME: use copyStorePaths()
554
555    PathSet valid = dstStore->queryValidPaths(closure);
556
557    if (valid.size() == closure.size()) return;
558
559    Paths sorted = srcStore->topoSortPaths(closure);
560
561    Paths missing;
562    for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
563        if (!valid.count(*i)) missing.push_back(*i);
564
565    printMsg(lvlDebug, format("copying %1% missing paths") % missing.size());
566
567    for (auto & i : missing)
568        copyStorePath(srcStore, dstStore, i, repair, dontCheckSigs);
569}
570
571
572ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
573{
574    ValidPathInfo info;
575    getline(str, info.path);
576    if (str.eof()) { info.path = ""; return info; }
577    if (hashGiven) {
578        string s;
579        getline(str, s);
580        info.narHash = parseHash(htSHA256, s);
581        getline(str, s);
582        if (!string2Int(s, info.narSize)) throw Error("number expected");
583    }
584    getline(str, info.deriver);
585    string s; int n;
586    getline(str, s);
587    if (!string2Int(s, n)) throw Error("number expected");
588    while (n--) {
589        getline(str, s);
590        info.references.insert(s);
591    }
592    if (!str || str.eof()) throw Error("missing input");
593    return info;
594}
595
596
597string showPaths(const PathSet & paths)
598{
599    string s;
600    for (auto & i : paths) {
601        if (s.size() != 0) s += ", ";
602        s += "‘" + i + "’";
603    }
604    return s;
605}
606
607
608std::string ValidPathInfo::fingerprint() const
609{
610    if (narSize == 0 || !narHash)
611        throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known")
612            % path);
613    return
614        "1;" + path + ";"
615        + printHashType(narHash.type) + ":" + printHash32(narHash) + ";"
616        + std::to_string(narSize) + ";"
617        + concatStringsSep(",", references);
618}
619
620
621void ValidPathInfo::sign(const SecretKey & secretKey)
622{
623    sigs.insert(secretKey.signDetached(fingerprint()));
624}
625
626
627bool ValidPathInfo::isContentAddressed(const Store & store) const
628{
629    auto warn = [&]() {
630        printError(format("warning: path ‘%s’ claims to be content-addressed but isn't") % path);
631    };
632
633    if (hasPrefix(ca, "text:")) {
634        auto hash = parseHash(std::string(ca, 5));
635        if (store.makeTextPath(storePathToName(path), hash, references) == path)
636            return true;
637        else
638            warn();
639    }
640
641    else if (hasPrefix(ca, "fixed:")) {
642        bool recursive = ca.compare(6, 2, "r:") == 0;
643        auto hash = parseHash(std::string(ca, recursive ? 8 : 6));
644        if (store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
645            return true;
646        else
647            warn();
648    }
649
650    return false;
651}
652
653
654size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const
655{
656    if (isContentAddressed(store)) return maxSigs;
657
658    size_t good = 0;
659    for (auto & sig : sigs)
660        if (checkSignature(publicKeys, sig))
661            good++;
662    return good;
663}
664
665
666bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
667{
668    return verifyDetached(fingerprint(), sig, publicKeys);
669}
670
671
672Strings ValidPathInfo::shortRefs() const
673{
674    Strings refs;
675    for (auto & r : references)
676        refs.push_back(baseNameOf(r));
677    return refs;
678}
679
680
681std::string makeFixedOutputCA(bool recursive, const Hash & hash)
682{
683    return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
684}
685
686
687}
688
689
690#include "local-store.hh"
691#include "remote-store.hh"
692
693
694namespace nix {
695
696
697RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
698
699
700ref<Store> openStore(const std::string & uri_)
701{
702    auto uri(uri_);
703    Store::Params params;
704    auto q = uri.find('?');
705    if (q != std::string::npos) {
706        for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
707            auto e = s.find('=');
708            if (e != std::string::npos)
709                params[s.substr(0, e)] = s.substr(e + 1);
710        }
711        uri = uri_.substr(0, q);
712    }
713    return openStore(uri, params);
714}
715
716ref<Store> openStore(const std::string & uri, const Store::Params & params)
717{
718    for (auto fun : *RegisterStoreImplementation::implementations) {
719        auto store = fun(uri, params);
720        if (store) return ref<Store>(store);
721    }
722
723    throw Error(format("don't know how to open Nix store ‘%s’") % uri);
724}
725
726
727StoreType getStoreType(const std::string & uri, const std::string & stateDir)
728{
729    if (uri == "daemon") {
730        return tDaemon;
731    } else if (uri == "local") {
732        return tLocal;
733    } else if (uri == "") {
734        if (access(stateDir.c_str(), R_OK | W_OK) == 0)
735            return tLocal;
736        else if (pathExists(settings.nixDaemonSocketFile))
737            return tDaemon;
738        else
739            return tLocal;
740    } else {
741        return tOther;
742    }
743}
744
745
746static RegisterStoreImplementation regStore([](
747    const std::string & uri, const Store::Params & params)
748    -> std::shared_ptr<Store>
749{
750    switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
751        case tDaemon:
752            return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
753        case tLocal:
754            return std::shared_ptr<Store>(std::make_shared<LocalStore>(params));
755        default:
756            return nullptr;
757    }
758});
759
760
761std::list<ref<Store>> getDefaultSubstituters()
762{
763    struct State {
764        bool done = false;
765        std::list<ref<Store>> stores;
766    };
767    static Sync<State> state_;
768
769    auto state(state_.lock());
770
771    if (state->done) return state->stores;
772
773    StringSet done;
774
775    auto addStore = [&](const std::string & uri) {
776        if (done.count(uri)) return;
777        done.insert(uri);
778        state->stores.push_back(openStore(uri));
779    };
780
781    Strings defaultSubstituters;
782    if (settings.nixStore == "/nix/store")
783        defaultSubstituters.push_back("https://cache.nixos.org/");
784
785    for (auto uri : settings.get("substituters", settings.get("binary-caches", defaultSubstituters)))
786        addStore(uri);
787
788    for (auto uri : settings.get("extra-binary-caches", Strings()))
789        addStore(uri);
790
791    state->done = true;
792
793    return state->stores;
794}
795
796
797void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute)
798{
799    PathSet valid = to->queryValidPaths(storePaths, substitute);
800
801    PathSet missing;
802    for (auto & path : storePaths)
803        if (!valid.count(path)) missing.insert(path);
804
805    std::string copiedLabel = "copied";
806
807    logger->setExpected(copiedLabel, missing.size());
808
809    ThreadPool pool;
810
811    processGraph<Path>(pool,
812        PathSet(missing.begin(), missing.end()),
813
814        [&](const Path & storePath) {
815            if (to->isValidPath(storePath)) return PathSet();
816            return from->queryPathInfo(storePath)->references;
817        },
818
819        [&](const Path & storePath) {
820            checkInterrupt();
821
822            if (!to->isValidPath(storePath)) {
823                Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath);
824
825                copyStorePath(from, to, storePath);
826
827                logger->incProgress(copiedLabel);
828            } else
829                logger->incExpected(copiedLabel, -1);
830        });
831
832    pool.process();
833}
834
835
836}
837