1#include "crypto.hh"
2#include "globals.hh"
3#include "store-api.hh"
4#include "util.hh"
5#include "nar-info-disk-cache.hh"
6#include "thread-pool.hh"
7#include "json.hh"
8#include "derivations.hh"
9
10#include <future>
11
12
13namespace nix {
14
15
16bool Store::isInStore(const Path & path) const
17{
18    return isInDir(path, storeDir);
19}
20
21
22bool Store::isStorePath(const Path & path) const
23{
24    return isInStore(path)
25        && path.size() >= storeDir.size() + 1 + storePathHashLen
26        && path.find('/', storeDir.size() + 1) == Path::npos;
27}
28
29
30void Store::assertStorePath(const Path & path) const
31{
32    if (!isStorePath(path))
33        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
34}
35
36
37Path Store::toStorePath(const Path & path) const
38{
39    if (!isInStore(path))
40        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
41    Path::size_type slash = path.find('/', storeDir.size() + 1);
42    if (slash == Path::npos)
43        return path;
44    else
45        return Path(path, 0, slash);
46}
47
48
49Path Store::followLinksToStore(const Path & _path) const
50{
51    Path path = absPath(_path);
52    while (!isInStore(path)) {
53        if (!isLink(path)) break;
54        string target = readLink(path);
55        path = absPath(target, dirOf(path));
56    }
57    if (!isInStore(path))
58        throw Error(format("path ‘%1%’ is not in the Nix store") % path);
59    return path;
60}
61
62
63Path Store::followLinksToStorePath(const Path & path) const
64{
65    return toStorePath(followLinksToStore(path));
66}
67
68
69string storePathToName(const Path & path)
70{
71    auto base = baseNameOf(path);
72    assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-'));
73    return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1);
74}
75
76
77string storePathToHash(const Path & path)
78{
79    auto base = baseNameOf(path);
80    assert(base.size() >= storePathHashLen);
81    return string(base, 0, storePathHashLen);
82}
83
84
85void checkStoreName(const string & name)
86{
87    string validChars = "+-._?=";
88    /* Disallow names starting with a dot for possible security
89       reasons (e.g., "." and ".."). */
90    if (string(name, 0, 1) == ".")
91        throw Error(format("illegal name: ‘%1%’") % name);
92    for (auto & i : name)
93        if (!((i >= 'A' && i <= 'Z') ||
94              (i >= 'a' && i <= 'z') ||
95              (i >= '0' && i <= '9') ||
96              validChars.find(i) != string::npos))
97        {
98            throw Error(format("invalid character ‘%1%’ in name ‘%2%’")
99                % i % name);
100        }
101}
102
103
104/* Store paths have the following form:
105
106   <store>/<h>-<name>
107
108   where
109
110   <store> = the location of the Nix store, usually /nix/store
111
112   <name> = a human readable name for the path, typically obtained
113     from the name attribute of the derivation, or the name of the
114     source file from which the store path is created.  For derivation
115     outputs other than the default "out" output, the string "-<id>"
116     is suffixed to <name>.
117
118   <h> = base-32 representation of the first 160 bits of a SHA-256
119     hash of <s>; the hash part of the store name
120
121   <s> = the string "<type>:sha256:<h2>:<store>:<name>";
122     note that it includes the location of the store as well as the
123     name to make sure that changes to either of those are reflected
124     in the hash (e.g. you won't get /nix/store/<h>-name1 and
125     /nix/store/<h>-name2 with equal hash parts).
126
127   <type> = one of:
128     "text:<r1>:<r2>:...<rN>"
129       for plain text files written to the store using
130       addTextToStore(); <r1> ... <rN> are the references of the
131       path.
132     "source"
133       for paths copied to the store using addToStore() when recursive
134       = true and hashAlgo = "sha256"
135     "output:<id>"
136       for either the outputs created by derivations, OR paths copied
137       to the store using addToStore() with recursive != true or
138       hashAlgo != "sha256" (in that case "source" is used; it's
139       silly, but it's done that way for compatibility).  <id> is the
140       name of the output (usually, "out").
141
142   <h2> = base-16 representation of a SHA-256 hash of:
143     if <type> = "text:...":
144       the string written to the resulting store path
145     if <type> = "source":
146       the serialisation of the path from which this store path is
147       copied, as returned by hashPath()
148     if <type> = "output:<id>":
149       for non-fixed derivation outputs:
150         the derivation (see hashDerivationModulo() in
151         primops.cc)
152       for paths copied by addToStore() or produced by fixed-output
153       derivations:
154         the string "fixed:out:<rec><algo>:<hash>:", where
155           <rec> = "r:" for recursive (path) hashes, or "" for flat
156             (file) hashes
157           <algo> = "md5", "sha1" or "sha256"
158           <hash> = base-16 representation of the path or flat hash of
159             the contents of the path (or expected contents of the
160             path for fixed-output derivations)
161
162   It would have been nicer to handle fixed-output derivations under
163   "source", e.g. have something like "source:<rec><algo>", but we're
164   stuck with this for now...
165
166   The main reason for this way of computing names is to prevent name
167   collisions (for security).  For instance, it shouldn't be feasible
168   to come up with a derivation whose output path collides with the
169   path for a copied source.  The former would have a <s> starting with
170   "output:out:", while the latter would have a <2> starting with
171   "source:".
172*/
173
174
175Path Store::makeStorePath(const string & type,
176    const Hash & hash, const string & name) const
177{
178    /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
179    string s = type + ":sha256:" + printHash(hash) + ":"
180        + storeDir + ":" + name;
181
182    checkStoreName(name);
183
184    return storeDir + "/"
185        + printHash32(compressHash(hashString(htSHA256, s), 20))
186        + "-" + name;
187}
188
189
190Path Store::makeOutputPath(const string & id,
191    const Hash & hash, const string & name) const
192{
193    return makeStorePath("output:" + id, hash,
194        name + (id == "out" ? "" : "-" + id));
195}
196
197
198Path Store::makeFixedOutputPath(bool recursive,
199    const Hash & hash, const string & name) const
200{
201    return hash.type == htSHA256 && recursive
202        ? makeStorePath("source", hash, name)
203        : makeStorePath("output:out", hashString(htSHA256,
204                "fixed:out:" + (recursive ? (string) "r:" : "") +
205                printHashType(hash.type) + ":" + printHash(hash) + ":"),
206            name);
207}
208
209
210Path Store::makeTextPath(const string & name, const Hash & hash,
211    const PathSet & references) const
212{
213    assert(hash.type == htSHA256);
214    /* Stuff the references (if any) into the type.  This is a bit
215       hacky, but we can't put them in `s' since that would be
216       ambiguous. */
217    string type = "text";
218    for (auto & i : references) {
219        type += ":";
220        type += i;
221    }
222    return makeStorePath(type, hash, name);
223}
224
225
226std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath,
227    bool recursive, HashType hashAlgo, PathFilter & filter) const
228{
229    Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
230    string name = baseNameOf(srcPath);
231    Path dstPath = makeFixedOutputPath(recursive, h, name);
232    return std::pair<Path, Hash>(dstPath, h);
233}
234
235
236Path Store::computeStorePathForText(const string & name, const string & s,
237    const PathSet & references) const
238{
239    return makeTextPath(name, hashString(htSHA256, s), references);
240}
241
242
243Store::Store(const Params & params)
244    : storeDir(get(params, "store", settings.nixStore))
245{
246}
247
248
249std::string Store::getUri()
250{
251    return "";
252}
253
254
255bool Store::isValidPath(const Path & storePath)
256{
257    auto hashPart = storePathToHash(storePath);
258
259    {
260        auto state_(state.lock());
261        auto res = state_->pathInfoCache.get(hashPart);
262        if (res) {
263            stats.narInfoReadAverted++;
264            return *res != 0;
265        }
266    }
267
268    if (diskCache) {
269        auto res = diskCache->lookupNarInfo(getUri(), hashPart);
270        if (res.first != NarInfoDiskCache::oUnknown) {
271            stats.narInfoReadAverted++;
272            auto state_(state.lock());
273            state_->pathInfoCache.upsert(hashPart,
274                res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
275            return res.first == NarInfoDiskCache::oValid;
276        }
277    }
278
279    bool valid = isValidPathUncached(storePath);
280
281    if (diskCache && !valid)
282        // FIXME: handle valid = true case.
283        diskCache->upsertNarInfo(getUri(), hashPart, 0);
284
285    return valid;
286}
287
288
289/* Default implementation for stores that only implement
290   queryPathInfoUncached(). */
291bool Store::isValidPathUncached(const Path & path)
292{
293    try {
294        queryPathInfo(path);
295        return true;
296    } catch (InvalidPath &) {
297        return false;
298    }
299}
300
301
302ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
303{
304    std::promise<ref<ValidPathInfo>> promise;
305
306    queryPathInfo(storePath,
307        [&](ref<ValidPathInfo> info) {
308            promise.set_value(info);
309        },
310        [&](std::exception_ptr exc) {
311            promise.set_exception(exc);
312        });
313
314    return promise.get_future().get();
315}
316
317
318void Store::queryPathInfo(const Path & storePath,
319    std::function<void(ref<ValidPathInfo>)> success,
320    std::function<void(std::exception_ptr exc)> failure)
321{
322    auto hashPart = storePathToHash(storePath);
323
324    try {
325
326        {
327            auto res = state.lock()->pathInfoCache.get(hashPart);
328            if (res) {
329                stats.narInfoReadAverted++;
330                if (!*res)
331                    throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
332                return success(ref<ValidPathInfo>(*res));
333            }
334        }
335
336        if (diskCache) {
337            auto res = diskCache->lookupNarInfo(getUri(), hashPart);
338            if (res.first != NarInfoDiskCache::oUnknown) {
339                stats.narInfoReadAverted++;
340                {
341                    auto state_(state.lock());
342                    state_->pathInfoCache.upsert(hashPart,
343                        res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
344                    if (res.first == NarInfoDiskCache::oInvalid ||
345                        (res.second->path != storePath && storePathToName(storePath) != ""))
346                        throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
347                }
348                return success(ref<ValidPathInfo>(res.second));
349            }
350        }
351
352    } catch (std::exception & e) {
353        return callFailure(failure);
354    }
355
356    queryPathInfoUncached(storePath,
357        [this, storePath, hashPart, success, failure](std::shared_ptr<ValidPathInfo> info) {
358
359            if (diskCache)
360                diskCache->upsertNarInfo(getUri(), hashPart, info);
361
362            {
363                auto state_(state.lock());
364                state_->pathInfoCache.upsert(hashPart, info);
365            }
366
367            if (!info
368                || (info->path != storePath && storePathToName(storePath) != ""))
369            {
370                stats.narInfoMissing++;
371                return failure(std::make_exception_ptr(InvalidPath(format("path ‘%s’ is not valid") % storePath)));
372            }
373
374            callSuccess(success, failure, ref<ValidPathInfo>(info));
375
376        }, failure);
377}
378
379
380PathSet Store::queryValidPaths(const PathSet & paths)
381{
382    struct State
383    {
384        size_t left;
385        PathSet valid;
386        std::exception_ptr exc;
387    };
388
389    Sync<State> state_(State{paths.size(), PathSet()});
390
391    std::condition_variable wakeup;
392
393    for (auto & path : paths)
394        queryPathInfo(path,
395            [path, &state_, &wakeup](ref<ValidPathInfo> info) {
396                auto state(state_.lock());
397                state->valid.insert(path);
398                assert(state->left);
399                if (!--state->left)
400                    wakeup.notify_one();
401            },
402            [path, &state_, &wakeup](std::exception_ptr exc) {
403                auto state(state_.lock());
404                try {
405                    std::rethrow_exception(exc);
406                } catch (InvalidPath &) {
407                } catch (...) {
408                    state->exc = exc;
409                }
410                assert(state->left);
411                if (!--state->left)
412                    wakeup.notify_one();
413            });
414
415    while (true) {
416        auto state(state_.lock());
417        if (!state->left) {
418            if (state->exc) std::rethrow_exception(state->exc);
419            return state->valid;
420        }
421        state.wait(wakeup);
422    }
423}
424
425
426/* Return a string accepted by decodeValidPathInfo() that
427   registers the specified paths as valid.  Note: it's the
428   responsibility of the caller to provide a closure. */
429string Store::makeValidityRegistration(const PathSet & paths,
430    bool showDerivers, bool showHash)
431{
432    string s = "";
433
434    for (auto & i : paths) {
435        s += i + "\n";
436
437        auto info = queryPathInfo(i);
438
439        if (showHash) {
440            s += printHash(info->narHash) + "\n";
441            s += (format("%1%\n") % info->narSize).str();
442        }
443
444        Path deriver = showDerivers ? info->deriver : "";
445        s += deriver + "\n";
446
447        s += (format("%1%\n") % info->references.size()).str();
448
449        for (auto & j : info->references)
450            s += j + "\n";
451    }
452
453    return s;
454}
455
456
457void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
458    bool includeImpureInfo, bool showClosureSize)
459{
460    auto jsonList = jsonOut.list();
461
462    for (auto storePath : storePaths) {
463        auto info = queryPathInfo(storePath);
464        storePath = info->path;
465
466        auto jsonPath = jsonList.object();
467        jsonPath
468            .attr("path", storePath)
469            .attr("narHash", info->narHash.to_string())
470            .attr("narSize", info->narSize);
471
472        {
473            auto jsonRefs = jsonPath.list("references");
474            for (auto & ref : info->references)
475                jsonRefs.elem(ref);
476        }
477
478        if (info->ca != "")
479            jsonPath.attr("ca", info->ca);
480
481        if (showClosureSize)
482            jsonPath.attr("closureSize", getClosureSize(storePath));
483
484        if (!includeImpureInfo) continue;
485
486        if (info->deriver != "")
487            jsonPath.attr("deriver", info->deriver);
488
489        if (info->registrationTime)
490            jsonPath.attr("registrationTime", info->registrationTime);
491
492        if (info->ultimate)
493            jsonPath.attr("ultimate", info->ultimate);
494
495        if (!info->sigs.empty()) {
496            auto jsonSigs = jsonPath.list("signatures");
497            for (auto & sig : info->sigs)
498                jsonSigs.elem(sig);
499        }
500    }
501}
502
503
504unsigned long long Store::getClosureSize(const Path & storePath)
505{
506    unsigned long long totalSize = 0;
507    PathSet closure;
508    computeFSClosure(storePath, closure, false, false);
509    for (auto & p : closure)
510        totalSize += queryPathInfo(p)->narSize;
511    return totalSize;
512}
513
514
515const Store::Stats & Store::getStats()
516{
517    {
518        auto state_(state.lock());
519        stats.pathInfoCacheSize = state_->pathInfoCache.size();
520    }
521    return stats;
522}
523
524
525void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
526    const Path & storePath, bool repair, bool dontCheckSigs)
527{
528    auto info = srcStore->queryPathInfo(storePath);
529
530    StringSink sink;
531    srcStore->narFromPath({storePath}, sink);
532
533    if (srcStore->isTrusted())
534        dontCheckSigs = true;
535
536    if (!info->narHash && dontCheckSigs) {
537        auto info2 = make_ref<ValidPathInfo>(*info);
538        info2->narHash = hashString(htSHA256, *sink.s);
539        info = info2;
540    }
541
542    dstStore->addToStore(*info, sink.s, repair, dontCheckSigs);
543}
544
545
546void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
547    const PathSet & storePaths, bool repair, bool dontCheckSigs)
548{
549    PathSet closure;
550    for (auto & path : storePaths)
551        srcStore->computeFSClosure(path, closure);
552
553    PathSet valid = dstStore->queryValidPaths(closure);
554
555    if (valid.size() == closure.size()) return;
556
557    Paths sorted = srcStore->topoSortPaths(closure);
558
559    Paths missing;
560    for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
561        if (!valid.count(*i)) missing.push_back(*i);
562
563    printMsg(lvlDebug, format("copying %1% missing paths") % missing.size());
564
565    for (auto & i : missing)
566        copyStorePath(srcStore, dstStore, i, repair, dontCheckSigs);
567}
568
569
570ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
571{
572    ValidPathInfo info;
573    getline(str, info.path);
574    if (str.eof()) { info.path = ""; return info; }
575    if (hashGiven) {
576        string s;
577        getline(str, s);
578        info.narHash = parseHash(htSHA256, s);
579        getline(str, s);
580        if (!string2Int(s, info.narSize)) throw Error("number expected");
581    }
582    getline(str, info.deriver);
583    string s; int n;
584    getline(str, s);
585    if (!string2Int(s, n)) throw Error("number expected");
586    while (n--) {
587        getline(str, s);
588        info.references.insert(s);
589    }
590    if (!str || str.eof()) throw Error("missing input");
591    return info;
592}
593
594
595string showPaths(const PathSet & paths)
596{
597    string s;
598    for (auto & i : paths) {
599        if (s.size() != 0) s += ", ";
600        s += "‘" + i + "’";
601    }
602    return s;
603}
604
605
606std::string ValidPathInfo::fingerprint() const
607{
608    if (narSize == 0 || !narHash)
609        throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known")
610            % path);
611    return
612        "1;" + path + ";"
613        + printHashType(narHash.type) + ":" + printHash32(narHash) + ";"
614        + std::to_string(narSize) + ";"
615        + concatStringsSep(",", references);
616}
617
618
619void ValidPathInfo::sign(const SecretKey & secretKey)
620{
621    sigs.insert(secretKey.signDetached(fingerprint()));
622}
623
624
625bool ValidPathInfo::isContentAddressed(const Store & store) const
626{
627    auto warn = [&]() {
628        printError(format("warning: path ‘%s’ claims to be content-addressed but isn't") % path);
629    };
630
631    if (hasPrefix(ca, "text:")) {
632        auto hash = parseHash(std::string(ca, 5));
633        if (store.makeTextPath(storePathToName(path), hash, references) == path)
634            return true;
635        else
636            warn();
637    }
638
639    else if (hasPrefix(ca, "fixed:")) {
640        bool recursive = ca.compare(6, 2, "r:") == 0;
641        auto hash = parseHash(std::string(ca, recursive ? 8 : 6));
642        if (store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
643            return true;
644        else
645            warn();
646    }
647
648    return false;
649}
650
651
652size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const
653{
654    if (isContentAddressed(store)) return maxSigs;
655
656    size_t good = 0;
657    for (auto & sig : sigs)
658        if (checkSignature(publicKeys, sig))
659            good++;
660    return good;
661}
662
663
664bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
665{
666    return verifyDetached(fingerprint(), sig, publicKeys);
667}
668
669
670Strings ValidPathInfo::shortRefs() const
671{
672    Strings refs;
673    for (auto & r : references)
674        refs.push_back(baseNameOf(r));
675    return refs;
676}
677
678
679std::string makeFixedOutputCA(bool recursive, const Hash & hash)
680{
681    return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string();
682}
683
684
685}
686
687
688#include "local-store.hh"
689#include "remote-store.hh"
690
691
692namespace nix {
693
694
695RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
696
697
698ref<Store> openStore(const std::string & uri_)
699{
700    auto uri(uri_);
701    Store::Params params;
702    auto q = uri.find('?');
703    if (q != std::string::npos) {
704        for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
705            auto e = s.find('=');
706            if (e != std::string::npos)
707                params[s.substr(0, e)] = s.substr(e + 1);
708        }
709        uri = uri_.substr(0, q);
710    }
711
712    for (auto fun : *RegisterStoreImplementation::implementations) {
713        auto store = fun(uri, params);
714        if (store) return ref<Store>(store);
715    }
716
717    throw Error(format("don't know how to open Nix store ‘%s’") % uri);
718}
719
720
721StoreType getStoreType(const std::string & uri, const std::string & stateDir)
722{
723    if (uri == "daemon") {
724        return tDaemon;
725    } else if (uri == "local") {
726        return tLocal;
727    } else if (uri == "") {
728        if (access(stateDir.c_str(), R_OK | W_OK) == 0)
729            return tLocal;
730        else if (pathExists(settings.nixDaemonSocketFile))
731            return tDaemon;
732        else
733            return tLocal;
734    } else {
735        return tOther;
736    }
737}
738
739
740static RegisterStoreImplementation regStore([](
741    const std::string & uri, const Store::Params & params)
742    -> std::shared_ptr<Store>
743{
744    switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
745        case tDaemon:
746            return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
747        case tLocal:
748            return std::shared_ptr<Store>(std::make_shared<LocalStore>(params));
749        default:
750            return nullptr;
751    }
752});
753
754
755std::list<ref<Store>> getDefaultSubstituters()
756{
757    struct State {
758        bool done = false;
759        std::list<ref<Store>> stores;
760    };
761    static Sync<State> state_;
762
763    auto state(state_.lock());
764
765    if (state->done) return state->stores;
766
767    StringSet done;
768
769    auto addStore = [&](const std::string & uri) {
770        if (done.count(uri)) return;
771        done.insert(uri);
772        state->stores.push_back(openStore(uri));
773    };
774
775    for (auto uri : settings.get("substituters", Strings()))
776        addStore(uri);
777
778    for (auto uri : settings.get("binary-caches", Strings()))
779        addStore(uri);
780
781    for (auto uri : settings.get("extra-binary-caches", Strings()))
782        addStore(uri);
783
784    state->done = true;
785
786    return state->stores;
787}
788
789
790void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
791{
792    if (substitute) {
793        /* Filter out .drv files (we don't want to build anything). */
794        PathSet paths2;
795        for (auto & path : storePaths)
796            if (!isDerivation(path)) paths2.insert(path);
797        unsigned long long downloadSize, narSize;
798        PathSet willBuild, willSubstitute, unknown;
799        to->queryMissing(PathSet(paths2.begin(), paths2.end()),
800            willBuild, willSubstitute, unknown, downloadSize, narSize);
801        /* FIXME: should use ensurePath(), but it only
802           does one path at a time. */
803        if (!willSubstitute.empty())
804            try {
805                to->buildPaths(willSubstitute);
806            } catch (Error & e) {
807                printMsg(lvlError, format("warning: %1%") % e.msg());
808            }
809    }
810
811    std::string copiedLabel = "copied";
812
813    logger->setExpected(copiedLabel, storePaths.size());
814
815    ThreadPool pool;
816
817    processGraph<Path>(pool,
818        PathSet(storePaths.begin(), storePaths.end()),
819
820        [&](const Path & storePath) {
821            return from->queryPathInfo(storePath)->references;
822        },
823
824        [&](const Path & storePath) {
825            checkInterrupt();
826
827            if (!to->isValidPath(storePath)) {
828                Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath);
829
830                copyStorePath(from, to, storePath);
831
832                logger->incProgress(copiedLabel);
833            } else
834                logger->incExpected(copiedLabel, -1);
835        });
836
837    pool.process();
838}
839
840
841}
842