1 /**
2     Validate and parse config files.
3 
4     Copyright: © 2019 Arne Ludwig <arne.ludwig@posteo.de>
5     License: Subject to the terms of the MIT license, as written in the
6              included LICENSE file.
7     Authors: Arne Ludwig <arne.ludwig@posteo.de>
8 */
9 module darg_plus.configfile;
10 
11 import darg :
12     Argument,
13     isArgumentHandler,
14     isOptionHandler,
15     Option,
16     OptionFlag;
17 import vibe.data.json : Json;
18 
19 ///
20 unittest
21 {
22     import darg : Multiplicity;
23     import vibe.data.json;
24 
25     auto config = serializeToJson([
26         "file": Json("/path/to/file"),
27         "more_files": serializeToJson([
28             "/path/to/file1",
29             "/path/to/file2",
30         ]),
31         "num": Json(42),
32         "verbose": Json(true),
33     ]);
34 
35     struct Options
36     {
37         @Argument("<in:file>")
38         string file;
39 
40         @Argument("<in:more_files>", Multiplicity.zeroOrMore)
41         string[] moreFiles;
42 
43         @Option("num")
44         size_t num;
45 
46         @Option("verbose")
47         OptionFlag verbose;
48     }
49 
50     auto options = parseConfig!Options(config);
51 
52     assert(options.file == "/path/to/file");
53     assert(options.moreFiles == [
54         "/path/to/file1",
55         "/path/to/file2",
56     ]);
57     assert(options.num == 42);
58     assert(options.verbose == true);
59 }
60 
61 ///
62 unittest
63 {
64     import vibe.data.json;
65 
66     auto config = serializeToJson([
67         "file": Json("/path/to/file"),
68         "num": Json(42),
69     ]);
70 
71     struct Options
72     {
73         @Argument("<in:file>")
74         string file;
75 
76         @Option("num")
77         size_t num = 1337;
78     }
79 
80     Options options;
81 
82     // Config values override default option values and arguments that are
83     // marked as empty:
84     options.file = configEmptyArgument;
85     assert(options.num == 1337);
86 
87     options = retroInitFromConfig(options, config);
88 
89     assert(options.file == "/path/to/file");
90     assert(options.num == 42);
91 
92     // Modified values, ie. given on the CLI, have precedence
93     // over config values:
94     options.file = "/path/from/cli";
95     options.num = 13;
96 
97     assert(options.file == "/path/from/cli");
98     assert(options.num == 13);
99 }
100 
101 /// String-type values equal to this string are considered empty.
102 enum configEmptyArgument = "-";
103 
104 /// Keys prefixed with this string are ignored.
105 enum configCommentPrefix = "//";
106 
107 /// Maximum size of a valid config file.
108 enum maxConfigSize = toBytes(256, SizeUnit.MiB);
109 
110 
111 /// Thrown if an error while handling config file occurs.
112 class ConfigFileException : Exception
113 {
114     string configKey;
115     Json configValue;
116 
117     /**
118         Params:
119             msg  = The message for the exception.
120             next = The previous exception in the chain of exceptions.
121             file = The file where the exception occurred.
122             line = The line number where the exception occurred.
123     */
124     this(string msg, Throwable next, string file = __FILE__,
125          size_t line = __LINE__) @nogc @safe pure nothrow
126     {
127         super(msg, file, line, next);
128     }
129 
130     /**
131         Params:
132             msg         = The message for the exception.
133             configKey   = Key of the erroneous config entry.
134             configValue = Value of the erroneous config entry.
135             file        = The file where the exception occurred.
136             line        = The line number where the exception occurred.
137             next        = The previous exception in the chain of exceptions, if any.
138     */
139     this(string msg, string configKey, Json configValue = Json.init, string file = __FILE__, size_t line = __LINE__,
140          Throwable next = null) @nogc @safe pure nothrow
141     {
142         super(msg, file, line, next);
143         this.configKey = configKey;
144         this.configValue = configValue;
145     }
146 
147     /**
148         Params:
149             msg         = The message for the exception.
150             configKey   = Key of the erroneous config entry.
151             configValue = Value of the erroneous config entry.
152             next        = The previous exception in the chain of exceptions.
153             file        = The file where the exception occurred.
154             line        = The line number where the exception occurred.
155     */
156     this(string msg, string configKey, Json configValue, Throwable next, string file = __FILE__,
157          size_t line = __LINE__) @nogc @safe pure nothrow
158     {
159         super(msg, file, line, next);
160         this.configKey = configKey;
161         this.configValue = configValue;
162     }
163 }
164 
165 T enforce(T)(
166     T value,
167     lazy string message,
168     lazy string configKey = null,
169     lazy Json configValue = Json.init,
170     string file = __FILE__,
171     size_t line = __LINE__,
172 )
173 {
174     static import std.exception;
175 
176     return std.exception.enforce(value, new ConfigFileException(
177         message,
178         configKey,
179         configValue,
180         file,
181         line,
182     ));
183 }
184 
185 
186 /// Retroactively initialize options from config.
187 Options retroInitFromConfig(Options)(ref Options options, in string configFile)
188 {
189     return retroInitFromConfig(options, parseConfig!Options(configFile));
190 }
191 
192 
193 /// ditto
194 Options retroInitFromConfig(Options)(ref Options options, in Json config)
195 {
196     import std.algorithm : all;
197     import std.format : format;
198     import std.math : isNaN;
199     import std.meta : Alias;
200     import std.range.primitives : ElementType;
201     import std.traits :
202         getUDAs,
203         isArray,
204         isFloatingPoint,
205         isSomeString,
206         isSomeString,
207         isStaticArray;
208 
209     enum defaultOptions = Options.init;
210     Options optionsFromConfig = parseConfig!Options(config);
211 
212     static foreach (member; __traits(allMembers, Options))
213     {{
214         alias symbol = Alias!(__traits(getMember, options, member));
215         enum isMemberAssignable = __traits(compiles,
216             __traits(getMember, options, member) = __traits(getMember, options, member)
217         );
218 
219         static if (isMemberAssignable)
220         {
221             alias Member = typeof(__traits(getMember, options, member));
222             enum unaryMixin(string template_) = format!template_(member);
223             enum binaryMixin(string template_) = format!template_(member, member);
224             alias assignConfigValue = () => mixin(binaryMixin!"options.%s = optionsFromConfig.%s");
225 
226             static if (getUDAs!(symbol, Argument).length > 0)
227             {
228                 static if (isSomeString!Member)
229                 {
230                     if (mixin(unaryMixin!"options.%s == configEmptyArgument"))
231                         assignConfigValue();
232                 }
233                 else static if (isArray!Member && isSomeString!(ElementType!Member))
234                 {
235                     if (mixin(unaryMixin!"options.%s.all!(v => v == configEmptyArgument)"))
236                         assignConfigValue();
237                 }
238             }
239             else
240             {
241                 static if (isStaticArray!Member || is(Member == class))
242                 {
243                     if (mixin(binaryMixin!"options.%s == defaultOptions.%s"))
244                         assignConfigValue();
245                 }
246                 else static if (isFloatingPoint!Member)
247                 {
248                     if (
249                         mixin(binaryMixin!"options.%s == defaultOptions.%s") ||
250                         (
251                             mixin(unaryMixin!"options.%s.isNaN") &&
252                             mixin(unaryMixin!"defaultOptions.%s.isNaN")
253                         )
254                     )
255                         assignConfigValue();
256                 }
257                 else
258                 {
259                     if (mixin(binaryMixin!"options.%s is defaultOptions.%s"))
260                         assignConfigValue();
261                 }
262             }
263         }
264     }}
265 
266     return options;
267 }
268 
269 
270 /// Initialize options using config.
271 Options parseConfig(Options)(in string configFile)
272 {
273     import vibe.data.json : parseJson;
274 
275     auto configContent = readConfigFile(configFile);
276     auto configValues = parseJson(
277         configContent,
278         null,
279         configFile,
280     );
281 
282     return parseConfig!Options(configValues);
283 }
284 
285 /// ditto
286 Options parseConfig(Options)(in Json config)
287 {
288     import std.meta : Alias;
289 
290     validateConfig!Options(config);
291 
292     Options options;
293 
294     foreach (member; __traits(allMembers, Options))
295     {
296         alias symbol = Alias!(__traits(getMember, options, member));
297         enum names = configNamesOf!symbol;
298 
299         static if (names.length > 0)
300         {
301             foreach (name; names)
302                 if (name in config)
303                     options.assignConfigValue!member(name, config[name]);
304         }
305     }
306 
307     return options;
308 }
309 
310 /// Validate config.
311 void validateConfigFile(Options)(in string configFile)
312 {
313     import vibe.data.json : parseJson;
314 
315     auto configContent = readConfigFile(configFile);
316     auto configValues = parseJson(
317         configContent,
318         null,
319         configFile,
320     );
321 
322     validateConfig!Options(configValues);
323 }
324 
325 /// ditto
326 void validateConfig(Options)(in Json config)
327 {
328     import std.algorithm : startsWith;
329     import std.format : format;
330     import std.meta : Alias;
331 
332     enforce(config.type == Json.Type.object, "config must contain a single object");
333 
334     configLoop: foreach (configKey, configValue; config.byKeyValue)
335     {
336         if (configKey.startsWith(configCommentPrefix))
337             continue;
338 
339         foreach (member; __traits(allMembers, Options))
340         {
341             alias symbol = Alias!(__traits(getMember, Options, member));
342             enum names = configNamesOf!symbol;
343 
344             static if (names.length > 0)
345             {
346                 alias SymbolType = typeof(__traits(getMember, Options, member));
347 
348                 foreach (name; names)
349                 {
350                     try
351                     {
352                         if (name == configKey)
353                         {
354                             cast(void) getConfigValue!SymbolType(configKey, configValue);
355                             continue configLoop;
356                         }
357                     }
358                     catch (Exception cause)
359                     {
360                         throw new ConfigFileException(
361                             format!"malformed config value `%s`: %s"(
362                                 configKey,
363                                 cause.msg,
364                             ),
365                             configKey,
366                             configValue,
367                             cause,
368                         );
369                     }
370                 }
371             }
372         }
373 
374         throw new ConfigFileException(
375             format!"invalid config key `%s`"(
376                 configKey,
377             ),
378             configKey,
379         );
380     }
381 }
382 
383 template configNamesOf(alias symbol)
384 {
385     import std.array : split;
386     import std.traits : getUDAs;
387 
388     alias optUDAs = getUDAs!(symbol, Option);
389     alias argUDAs = getUDAs!(symbol, Argument);
390 
391     static if (argUDAs.length > 0)
392         enum argName = argUDAs[0].name.split(":")[$ - 1][0 .. $ - 1];
393 
394     static if (optUDAs.length > 0 && argUDAs.length > 0)
395     {
396         enum configNamesOf = optUDAs[0].names ~ argName;
397     }
398     else static if (optUDAs.length > 0)
399     {
400         enum configNamesOf = optUDAs[0].names;
401     }
402     else static if (argUDAs.length > 0)
403     {
404         enum configNamesOf = [argName];
405     }
406     else
407     {
408         enum string[] configNamesOf = [];
409     }
410 }
411 
412 void assignConfigValue(string member, Options)(ref Options options, string configKey, Json configValue)
413 {
414     import std.conv : to;
415     import std.traits : isAssignable;
416 
417     alias SymbolType = typeof(__traits(getMember, options, member));
418 
419     static if (isOptionHandler!SymbolType)
420     {
421         if (configValue.type == Json.Type.int_)
422             foreach (i; 0 .. configValue.get!ulong)
423                 __traits(getMember, options, member)();
424         else if (configValue.type == Json.Type.bool_)
425         {
426             if (configValue.get!bool)
427                 __traits(getMember, options, member)();
428         }
429         else
430             throw new ConfigFileException(
431                 "Got JSON of type " ~ configValue.type.to!string ~
432                 ", expected int_ or bool_.",
433                 configKey,
434                 configValue,
435             );
436     }
437     else static if (isArgumentHandler!SymbolType)
438     {
439         if (configValue.type == Json.Type.array)
440             foreach (item; configValue.get!(Json[]))
441                 __traits(getMember, options, member)(item.get!string);
442         else if (configValue.type == Json.Type..string)
443             __traits(getMember, options, member)(configValue.get!string);
444         else
445             throw new ConfigFileException(
446                 "Got JSON of type " ~ configValue.type.to!string ~
447                 ", expected array or string_.",
448                 configKey,
449                 configValue,
450             );
451     }
452     else static if (isAssignable!SymbolType)
453     {
454         __traits(getMember, options, member) = getConfigValue!SymbolType(configKey, configValue);
455     }
456 }
457 
458 auto getConfigValue(SymbolType)(string configKey, Json configValue)
459 {
460     import std.conv : to;
461     import std.range.primitives : ElementType;
462     import std.traits :
463         isArray,
464         isDynamicArray,
465         isFloatingPoint,
466         isIntegral,
467         isSomeString,
468         isUnsigned;
469 
470     static if (is(SymbolType == OptionFlag))
471         return configValue.get!bool.to!SymbolType;
472     else static if (is(SymbolType == enum))
473         return configValue.get!string.to!SymbolType;
474     else static if (is(SymbolType == OptionFlag) || is(SymbolType : bool))
475         return configValue.get!bool.to!SymbolType;
476     else static if (isFloatingPoint!SymbolType)
477         return configValue.get!double.to!SymbolType;
478     else static if (isIntegral!SymbolType && isUnsigned!SymbolType)
479         return configValue.get!ulong.to!SymbolType;
480     else static if (isIntegral!SymbolType && !isUnsigned!SymbolType)
481         return configValue.get!long.to!SymbolType;
482     else static if (isSomeString!SymbolType)
483     {
484         if (configValue.type == Json.Type..string)
485             return configValue.get!string.to!SymbolType;
486         else if (configValue.type == Json.Type.null_)
487             return null;
488         else
489             throw new ConfigFileException(
490                 "Got JSON of type " ~ configValue.type.to!string ~
491                 ", expected string or null_.",
492                 configKey,
493                 configValue,
494             );
495     }
496     else static if (isArray!SymbolType)
497     {
498         SymbolType value;
499 
500         static if (isDynamicArray!SymbolType)
501             value.length = configValue.length;
502         else
503             enforce(
504                 configValue.length == value.length,
505                 "array must have " ~ value.length ~ " elements",
506                 configKey,
507                 configValue,
508             );
509 
510         foreach (size_t i, configElement; configValue.get!(Json[]))
511             value[i] = getConfigValue!(ElementType!SymbolType)(configKey, configElement);
512 
513         return value;
514     }
515 }
516 
517 string readConfigFile(in string configFileName)
518 {
519     import std.stdio : File;
520     import std.format : format;
521 
522     auto configFile = File(configFileName, "r");
523     auto configFileSize = configFile.size;
524 
525     enforce(
526         configFileSize <= maxConfigSize,
527         format!"config file is too large; must be <= %.2f %s"(fromBytes(maxConfigSize).expand),
528     );
529 
530     auto configContent = configFile.rawRead(new char[configFileSize]);
531 
532     return cast(string) configContent;
533 }
534 
535 /// Units for bytes.
536 enum SizeUnit
537 {
538     B,
539     KiB,
540     MiB,
541     GiB,
542     TiB,
543     PiB,
544     EiB,
545     ZiB,
546     YiB,
547 }
548 
549 /// Convert a value and unit to number of bytes.
550 auto toBytes(in size_t value, in SizeUnit unit)
551 {
552     return value * sizeUnitBase^^unit;
553 }
554 
555 /// Convert bytes to
556 auto fromBytes(in size_t bytes)
557 {
558     import std.conv : to;
559     import std.typecons : tuple;
560     import std.traits : EnumMembers;
561 
562     alias convertToUnit = exp => tuple!("value", "unit")(
563         bytes.to!double / (sizeUnitBase^^exp),
564         exp,
565     );
566 
567     foreach (exp; EnumMembers!SizeUnit)
568     {
569         if (bytes <= sizeUnitBase^^exp)
570             return convertToUnit(exp);
571     }
572 
573     return convertToUnit(SizeUnit.max);
574 }
575 
576 private enum size_t sizeUnitBase = 2^^10;