1 module mir.xlsx; 2 3 import core.time : Duration; 4 import std.algorithm.iteration : filter, map; 5 import std.algorithm.searching : all, canFind, startsWith; 6 import std.algorithm.sorting : sort; 7 import std.array : array, empty, front, popFront, Appender; 8 import std.conv : to; 9 import std.datetime : DateTime, Date, TimeOfDay; 10 import std.datetime.stopwatch : StopWatch, AutoStart; 11 import std.exception : enforce; 12 import std.file : read, exists, readText; 13 import std.format : format; 14 import std.traits : isIntegral, isFloatingPoint, isSomeString; 15 import std.typecons : Nullable, nullable; 16 import std.zip; 17 import mir.algebraic : Algebraic, Variant; 18 import mir.reflection : reflectIgnore, ReflectDoc; 19 import mir.ion.value : IonNull; 20 import mir.timestamp : Timestamp; 21 import dxml.dom : DOMEntity, EntityType, parseDOM; 22 import dxml.util : decodeXML; // TODO: Replace with parseXML 23 import dxml.parser : parseXML; // TODO: Use instead of decodeXML 24 25 debug import std.stdio; 26 27 alias SILignore = reflectIgnore!"SIL"; 28 alias SILdoc = ReflectDoc!"SIL"; 29 30 // disabled for now for faster builds 31 // version = ctRegex_test; 32 33 version(mir_profileGC) 34 enum runCount = 1; 35 else version(mir_benchmark) 36 enum runCount = 10; 37 38 version(mir_benchmark) { 39 enum tme = true; // time me 40 import std.stdio; 41 } 42 else 43 enum tme = false; // don’t time 44 45 /** Row or column offset precision. 46 * 47 * Excel has a limit of 1_048_576 rows and 16_384 columns per sheet so use 48 * 32-bit precision now. 49 * 50 * Uses alias instead of sub-type for backwards compatibility in for instance ’Pos’. 51 */ 52 alias Offset = uint; 53 54 /** Row width or column height precision. 55 * 56 * Uses alias instead of sub-type for backwards compatibility in for instance ’Pos’. 57 */ 58 alias Length = uint; 59 60 /** Row offset, starting at 0 for top-most row. 61 * 62 * Defaults to uint.max to indicate not yet initialized (undefined). 63 * 64 * Uses sub-type instead of alias for type-safe API requiring explicit conversions. 65 */ 66 struct RowOffset { 67 enum undefined = 68 1_048_576; ///< Excel has a limit of 1_048_576 rows so make the 1_048_576+1:th represent undefinedness. 69 enum max = undefined - 1; 70 Offset value = undefined; 71 alias value this; 72 this(Offset value) @safe pure nothrow @nogc { 73 this.value = value; 74 } 75 76 this(size_t value) @safe pure { 77 this.value = value.to!Offset; 78 } 79 80 bool isDefined() const @safe pure nothrow @nogc { 81 return value != undefined; 82 } 83 84 invariant(value <= undefined); 85 } 86 87 /** Column offset, starting at 0 for left-most column. 88 * 89 * Defaults to uint.max to indicate not yet initialized (undefined). 90 * 91 * Uses sub-type instead of alias for type-safe API requiring explicit conversions. 92 */ 93 struct ColumnOffset { 94 enum undefined = 95 16_384; ///< Excel has a limit of 16_384 rows so make the 16_384+1:th represent undefinedness. 96 enum max = undefined - 1; 97 Offset value = undefined; 98 alias value this; 99 this(Offset value) @safe pure nothrow @nogc { 100 this.value = value; 101 } 102 103 this(size_t value) @safe pure { 104 this.value = value.to!Offset; 105 } 106 107 bool isDefined() const @safe pure nothrow @nogc { 108 return value != undefined; 109 } 110 111 invariant(value <= undefined); 112 } 113 114 alias ColOffset = ColumnOffset; 115 116 /** Row width in number of rows. 117 */ 118 struct RowWidth { 119 enum undefined = RowOffset.undefined + 1; 120 enum max = undefined - 1; 121 Length value = undefined; 122 alias value this; 123 this(Offset value) @safe pure nothrow @nogc { 124 this.value = value; 125 } 126 127 this(size_t value) @safe pure { 128 this.value = value.to!Length; 129 } 130 131 bool isDefined() const @safe pure nothrow @nogc { 132 return value != undefined; 133 } 134 135 invariant(value <= undefined); 136 } 137 138 /** Column height in number of columns. 139 */ 140 struct ColumnHeight { 141 enum undefined = ColumnOffset.undefined + 1; 142 enum max = undefined - 1; 143 Length value = undefined; 144 alias value this; 145 this(Offset value) @safe pure nothrow @nogc { 146 this.value = value; 147 } 148 149 this(size_t value) @safe pure { 150 this.value = value.to!Length; 151 } 152 153 bool isDefined() const @safe pure nothrow @nogc { 154 return value != undefined; 155 } 156 157 invariant(value <= undefined); 158 } 159 160 alias ColHeight = ColumnHeight; 161 162 /** SparseCell position offset, starting at [0,0] for top-left cell. 163 See: https://support.socrata.com/hc/en-us/articles/115005306167-Limitations-of-Excel-and-CSV-Downloads 164 */ 165 struct Position { 166 RowOffset row; 167 ColumnOffset column; 168 169 alias col = column; 170 /// Position of top-left corner of sheet. 171 static Position origin() @safe pure nothrow @nogc { 172 return typeof(return)(RowOffset(0), ColumnOffset(0)); 173 } 174 175 bool isDefined() const @safe pure nothrow @nogc { 176 return row.isDefined && column.isDefined; 177 } 178 } 179 180 /** Excel cell address in the format "A1", "A2", .... 181 */ 182 struct Address { 183 string value; 184 alias value this; 185 } 186 187 /** Excel table start position/address. 188 * 189 * TODO: Should we wrap this algebraic in a struct 190 */ 191 alias Start = Algebraic!(Address, Position); 192 193 /** Excel table extents as width * height. 194 */ 195 struct Extent { 196 RowWidth width; 197 ColumnHeight height; 198 } 199 200 /** Excel sheet region. 201 */ 202 struct Region { 203 Start start; 204 Extent extent; 205 } 206 207 /** SparseCell value without `null` state. 208 */ 209 alias SparseValue = Variant!(bool, Timestamp, double, long, string); 210 211 /** SparseCell value with `null` state. 212 * 213 * The state `null` is needed because dense table cells may be uninitialized 214 * when copied from the sparse representation in `Sheet.cells`. 215 */ 216 alias DenseValue = Variant!(typeof(null), bool, Timestamp, double, long, string); // TODO: reuse `SparseValue` 217 218 /// Sheet cell that holds `position` for use in sparse table storage. 219 struct SparseCell { 220 this(RowOffset row, string t, string r, string v, string formula, 221 string xmlValue, Position position) @safe pure { 222 this.row = row; 223 this.t = t; 224 this.r = r; 225 this.v = v; 226 this.formula = formula; 227 this.xmlValue = xmlValue; 228 this.position = position; 229 this.value = xmlValue; 230 } 231 232 @SILignore 233 RowOffset row; ///< Row. row[r] 234 235 @SILignore 236 string t; ///< XML Type attribute 237 238 @SILignore 239 string r; // c[r] 240 241 @SILignore 242 string v; // value or ptr 243 244 @SILignore 245 string formula; // formula 246 247 @SILignore 248 string xmlValue; ///< Value stored in cell. TODO: convert this 249 250 @SILignore 251 Position position; ///< Position of cell. 252 253 SparseValue value; ///< Decoded value. 254 } 255 256 /// Sheet cell that doesn’t need to hold position in dense table storage. 257 struct DenseCell { 258 this(DenseValue value, string xmlValue, string formula) @safe pure nothrow @nogc { 259 this.value = value; 260 this.xmlValue = xmlValue; 261 this.formula = formula; 262 } 263 this(SparseValue value, string xmlValue, string formula) @safe pure nothrow @nogc { 264 this.value = value; 265 this.xmlValue = xmlValue; 266 this.formula = formula; 267 } 268 DenseValue value; ///< Decoded cell value. 269 string xmlValue; // TODO: remove 270 @SILignore 271 string formula; ///< SparseCell formula. 272 } 273 274 /** Excel table as a 2-dimensional dense array. 275 * 276 * Ref: https://en.wikipedia.org/wiki/Array_(data_type) 277 */ 278 struct DenseTable { 279 inout(DenseCell) opIndex( 280 in RowOffset rowOffset, 281 in ColumnOffset columnOffset 282 ) inout scope @safe pure nothrow @nogc { 283 version(LDC) 284 pragma(inline, true); 285 return _cells[rowOffset * extent.width + columnOffset]; 286 } 287 288 // Support for `x..y` notation in slicing operator for the given dimension. 289 version(none) 290 Offset[2] opSlice(size_t dim)(Offset start, Offset end) 291 if (dim >= 0 && dim < 2) 292 in(start >= 0 && end <= this.opDollar!dim) { 293 pragma(inline, true); 294 return [start, end]; 295 } 296 297 @SILignore 298 inout(DenseCell)[] cells() inout @safe pure nothrow @nogc { 299 pragma(inline, true); 300 return _cells; 301 } 302 303 @SILignore 304 auto byRow() const scope @safe pure nothrow { 305 struct Result { // TODO: reuse ndslice range instead? 306 bool empty() const @property @safe pure nothrow @nogc { 307 return _rowOffset == _denseTable.extent.height; 308 } 309 310 inout(const(DenseCell))[] front() inout @property 311 @safe pure nothrow @nogc { 312 assert(!empty); 313 return _denseTable._cells[_rowOffset * _denseTable.extent.width 314 .. (_rowOffset + 1) * _denseTable.extent.width]; 315 } 316 317 void popFront() @safe pure nothrow @nogc { 318 assert(!empty); 319 _rowOffset += 1; 320 } 321 322 private: 323 const DenseTable _denseTable; 324 RowOffset _rowOffset = RowOffset(0); 325 } 326 327 return Result(this, RowOffset(0)); 328 } 329 330 const(DenseCell)[][] rows() const return scope @safe pure nothrow { 331 return byRow().array; 332 } 333 334 @SILignore 335 DenseCell[] 336 _cells; // TODO: use immutable(DenseCell)* _cells instead because length is same as extents.width*extents.height 337 alias _cells this; 338 @SILignore 339 Extent extent; 340 invariant { 341 assert(_cells.length == extent.width * extent.height); 342 } 343 } 344 345 /// Sheet. 346 struct Sheet { 347 import std.ascii : toUpper; 348 349 this(string name, SparseCell[] cells, 350 Extent extent) @safe pure nothrow @nogc { 351 this.name = name; 352 this._cells = cells; 353 this.extent = extent; 354 } 355 356 @SILignore 357 inout(SparseCell)[] 358 cells() inout @property return scope @safe pure nothrow @nogc { 359 return _cells; 360 } 361 362 const string name; ///< Name of sheet. 363 364 @SILignore 365 private SparseCell[] _cells; ///< Cells of sheet. 366 367 @SILignore 368 inout(DenseTable) denseTable() inout @property 369 @trusted pure nothrow { // TODO: make mutable? 370 pragma(inline, true); 371 if (_denseTable is _denseTable.init) 372 // TODO: remove this if and when we decide if densetable should be removed: 373 *(cast(DenseTable*) &_denseTable) = (cast() this).makeDenseTable(); 374 return _denseTable; 375 } 376 alias table = denseTable; 377 378 @SILignore 379 private DenseTable makeDenseTable() const @trusted pure nothrow { 380 auto tab = new DenseCell[](extent.width * extent.height); 381 foreach (const ref cell; cells) 382 tab[cell.position.row * extent.width + cell.position.col] = DenseCell(cell.value, cell.xmlValue, cell.formula); 383 return typeof(return)(tab, extent); 384 } 385 386 @SILignore 387 private DenseTable _denseTable; 388 389 @SILignore 390 const Extent extent; 391 392 string toString() const @property scope @safe pure { 393 import std.array : appender; 394 auto result = appender!(typeof(return)); 395 toString(result); 396 return result.data[]; 397 } 398 399 void toString(Sink)(ref scope Sink sink) const scope { 400 import std.algorithm.comparison : max; 401 scope lens = new size_t[](extent.width); 402 scope tab = makeDenseTable(); // TODO: avoid this allocation 403 foreach (const ref row; tab.byRow) 404 foreach (const idx, const ref DenseCell cell; row) 405 lens[idx] = 406 max(lens[idx], 407 cell.value.to!string.length); // TODO: use cell.toString 408 409 import std.format : formattedWrite; 410 foreach (const ref row; tab.byRow) { 411 foreach (const idx, const ref DenseCell cell; row) 412 sink.formattedWrite("%*s, ", lens[idx] + 1, 413 cell.value.to!string); // TODO: use cell.toString 414 sink.formattedWrite("\n"); 415 } 416 } 417 418 @SILignore: 419 420 ColumnRange getColumn(ColumnOffset col, RowOffset startRow, RowOffset endRow) return scope @safe { 421 return typeof(return)(this, col, startRow, endRow); 422 } 423 424 RowRange getRow(RowOffset row, ColumnOffset startColumn, ColumnOffset endColumn) return scope @safe { 425 return typeof(return)(this, row, startColumn, endColumn); 426 } 427 } 428 429 /// 430 struct RowRange { 431 Sheet sheet; 432 const RowOffset row; 433 const ColumnOffset startColumn; 434 const ColumnOffset endColumn; 435 ColumnOffset cur; 436 437 this(Sheet sheet, RowOffset row, ColumnOffset startColumn, 438 ColumnOffset endColumn) pure nothrow /* @nogc */ @safe { 439 this.sheet = sheet; 440 this.row = row; 441 this.startColumn = startColumn; 442 this.endColumn = endColumn; 443 this.cur = this.startColumn; 444 } 445 446 bool empty() const @property pure nothrow @nogc @safe { 447 return this.cur >= this.endColumn; 448 } 449 450 void popFront() pure nothrow @nogc @safe { 451 ++this.cur; 452 } 453 454 inout(typeof(this)) save() inout @property pure nothrow @nogc @safe { 455 return this; 456 } 457 458 inout(DenseCell) front() inout @property pure nothrow /* @nogc */ @safe { 459 return this.sheet.denseTable[row, cur]; 460 } 461 } 462 463 /// 464 struct ColumnRange { 465 Sheet sheet; 466 const ColumnOffset col; 467 const RowOffset startRow; 468 const RowOffset endRow; 469 RowOffset cur; 470 471 this(Sheet sheet, ColumnOffset col, RowOffset startRow, 472 RowOffset endRow) @safe { 473 this.sheet = sheet; 474 this.col = col; 475 this.startRow = startRow; 476 this.endRow = endRow; 477 this.cur = this.startRow; 478 } 479 480 bool empty() const @property pure nothrow @nogc @safe { 481 return this.cur >= this.endRow; 482 } 483 484 void popFront() pure nothrow @nogc @safe { 485 ++this.cur; 486 } 487 488 inout(typeof(this)) save() inout @property pure nothrow @nogc @safe { 489 return this; 490 } 491 492 inout(DenseCell) front() inout @property pure nothrow /* @nogc */ @safe { 493 return this.sheet.denseTable[cur, col]; 494 } 495 } 496 497 version(mir_test) 498 @safe 499 unittest { 500 import std.range : isForwardRange; 501 static assert(isForwardRange!ColumnRange); 502 static assert(isForwardRange!RowRange); 503 } 504 505 Date longToDate(long d) @safe { 506 // modifed from https://www.codeproject.com/Articles/2750/ 507 // Excel-Serial-Date-to-Day-Month-Year-and-Vice-Versa 508 509 // Excel/Lotus 123 have a bug with 29-02-1900. 1900 is not a 510 // leap year, but Excel/Lotus 123 think it is... 511 if (d == 60) { 512 return Date(1900, 2, 29); 513 } else if (d < 60) { 514 // Because of the 29-02-1900 bug, any serial date 515 // under 60 is one off... Compensate. 516 ++d; 517 } 518 519 // Modified Julian to DMY calculation with an addition of 2415019 520 int l = cast(int) d + 68569 + 2415019; 521 const int n = int((4 * l) / 146097); 522 l = l - int((146097 * n + 3) / 4); 523 const int i = int((4000 * (l + 1)) / 1461001); 524 l = l - int((1461 * i) / 4) + 31; 525 const int j = int((80 * l) / 2447); 526 const nDay = l - int((2447 * j) / 80); 527 l = int(j / 11); 528 const int nMonth = j + 2 - (12 * l); 529 const int nYear = 100 * (n - 49) + i + l; 530 return Date(nYear, nMonth, nDay); 531 } 532 533 long dateToLong(Date d) @safe { 534 // modifed from https://www.codeproject.com/Articles/2750/ 535 // Excel-Serial-Date-to-Day-Month-Year-and-Vice-Versa 536 537 // Excel/Lotus 123 have a bug with 29-02-1900. 1900 is not a 538 // leap year, but Excel/Lotus 123 think it is... 539 if (d.day == 29 && d.month == 2 && d.year == 1900) { 540 return 60; 541 } 542 543 // DMY to Modified Julian calculated with an extra subtraction of 2415019. 544 long nSerialDate = int( 545 (1461 * (d.year + 4800 + int((d.month - 14) / 12))) / 4) 546 + int((367 * (d.month - 2 - 12 * ((d.month - 14) / 12))) / 12) 547 - int((3 * (int((d.year + 4900 + int((d.month - 14) / 12)) / 100))) / 4) 548 + d.day - 2415019 - 32075; 549 550 if (nSerialDate < 60) { 551 // Because of the 29-02-1900 bug, any serial date 552 // under 60 is one off... Compensate. 553 nSerialDate--; 554 } 555 556 return nSerialDate; 557 } 558 559 version(mir_test) 560 @safe 561 unittest { 562 auto ds = [Date(1900, 2, 1), Date(1901, 2, 28), Date(2019, 06, 05)]; 563 foreach (const d; ds) { 564 const long l = dateToLong(d); 565 const Date r = longToDate(l); 566 assert(r == d); 567 } 568 } 569 570 TimeOfDay doubleToTimeOfDay(double s) @safe { 571 import core.stdc.math : lround; 572 const double secs = (24.0 * 60.0 * 60.0) * s; 573 574 // TODO not one-hundred my lround is needed 575 const int secI = to!int(lround(secs)); 576 577 return TimeOfDay(secI / 3600, (secI / 60) % 60, secI % 60); 578 } 579 580 double timeOfDayToDouble(TimeOfDay tod) @safe { 581 const long h = tod.hour * 60 * 60; 582 const long m = tod.minute * 60; 583 const long s = tod.second; 584 return (h + m + s) / (24.0 * 60.0 * 60.0); 585 } 586 587 version(mir_test) 588 @safe 589 unittest { 590 auto tods = 591 [TimeOfDay(23, 12, 11), TimeOfDay(11, 0, 11), TimeOfDay(0, 0, 0), 592 TimeOfDay(0, 1, 0), TimeOfDay(23, 59, 59), TimeOfDay(0, 0, 0)]; 593 foreach (const tod; tods) { 594 const double d = timeOfDayToDouble(tod); 595 assert(d <= 1.0); 596 TimeOfDay r = doubleToTimeOfDay(d); 597 assert(r == tod); 598 } 599 } 600 601 double datetimeToDouble(DateTime dt) @safe { 602 const double d = dateToLong(dt.date); 603 const double t = timeOfDayToDouble(dt.timeOfDay); 604 return d + t; 605 } 606 607 DateTime doubleToDateTime(double d) @safe { 608 long l = cast(long) d; 609 Date dt = longToDate(l); 610 TimeOfDay t = doubleToTimeOfDay(d - l); 611 return DateTime(dt, t); 612 } 613 614 version(mir_test) 615 @safe 616 unittest { 617 auto ds = [Date(1900, 2, 1), Date(1901, 2, 28), Date(2019, 06, 05)]; 618 auto tods = 619 [TimeOfDay(23, 12, 11), TimeOfDay(11, 0, 11), TimeOfDay(0, 0, 0), 620 TimeOfDay(0, 1, 0), TimeOfDay(23, 59, 59), TimeOfDay(0, 0, 0)]; 621 foreach (const d; ds) { 622 foreach (const tod; tods) { 623 DateTime dt = DateTime(d, tod); 624 double dou = datetimeToDouble(dt); 625 626 Date rd = longToDate(cast(long) dou); 627 assert(rd == d); 628 629 double rest = dou - cast(long) dou; 630 TimeOfDay rt = doubleToTimeOfDay(dou - cast(long) dou); 631 assert(rt == tod); 632 633 DateTime r = doubleToDateTime(dou); 634 assert(r == dt); 635 } 636 } 637 } 638 639 Date stringToDate(string s) @safe { 640 import std.array : split; 641 import std.string : indexOf; 642 643 if (s.indexOf('/') != -1) { 644 auto sp = s.split('/'); 645 enforce(sp.length == 3, format("[%s]", sp)); 646 return Date(to!int(sp[2]), to!int(sp[1]), to!int(sp[0])); 647 } else { 648 return longToDate(to!long(s)); 649 } 650 } 651 652 bool tryConvertTo(T, S)(S var) { 653 return !(tryConvertToImpl!T(DenseValue(var)).isNull()); 654 } 655 656 Nullable!(T) tryConvertToImpl(T)(DenseValue var) { 657 try { 658 return nullable(convertTo!T(var)); 659 } catch (Exception e) { 660 return Nullable!T(); 661 } 662 } 663 664 T convertTo(T)(string var) @safe { 665 import std.math : lround; 666 static if (isSomeString!T) { 667 return to!T(var); 668 } else static if (is(T == bool)) { 669 return var == "1"; 670 } else static if (isIntegral!T) { 671 return to!T(var); 672 } else static if (isFloatingPoint!T) { 673 return to!T(var); 674 } else static if (is(T == DateTime)) { 675 if (var.canConvertToLong()) { 676 return doubleToDateTime(to!long(var)); 677 } else if (var.canConvertToDouble()) { 678 return doubleToDateTime(to!double(var)); 679 } 680 681 enforce(false, "Can not convert '" ~ var ~ "' to a DateTime"); 682 assert(false, "Unreachable"); 683 } else static if (is(T == Date)) { 684 if (var.canConvertToLong()) { 685 return longToDate(to!long(var)); 686 } else if (var.canConvertToDouble()) { 687 return longToDate(lround(to!double(var))); 688 } 689 690 return stringToDate(var); 691 } else static if (is(T == TimeOfDay)) { 692 const double l = to!double(var); 693 return doubleToTimeOfDay(l - cast(long) l); 694 } else { 695 static assert(false, T.stringof ~ " not supported"); 696 } 697 } 698 699 private ZipArchive readFile(in string filename) @trusted { 700 enforce(exists(filename), "File with name " ~ filename ~ " does not exist"); 701 return new typeof(return)(read(filename)); 702 } 703 704 private static immutable workbookXMLPath = "xl/workbook.xml"; 705 private static immutable sharedStringXMLPath = "xl/sharedStrings.xml"; 706 private static immutable relsXMLPath = "xl/_rels/workbook.xml.rels"; 707 708 private static expandTrusted(ZipArchive za, 709 ArchiveMember de) @trusted /* TODO: pure */ { 710 static if (tme) 711 auto sw = StopWatch(AutoStart.yes); 712 auto ret = za.expand(de); 713 static if (tme) 714 writeln("expand length:", ret.length, " took: ", sw.peek()); 715 return ret; 716 } 717 718 struct Relationships { 719 string id; 720 string file; 721 } 722 723 alias RelationshipsById = Relationships[string]; 724 725 /// Workbook. 726 struct Workbook { 727 @SILignore 728 static typeof(this) fromFile(in string filename) @trusted { 729 return typeof(return)(filename, new ZipArchive(read(filename))); 730 } 731 732 @SILignore 733 static typeof(this) fromBytes(void[] buffer) @trusted { 734 return typeof(return)("", new ZipArchive(buffer)); 735 } 736 737 @SILignore 738 package SheetNameId[] sheetNameIds() @safe /* TODO: pure */ { 739 auto dom = workbookDOM(); 740 if (dom.children.length != 1) 741 return []; 742 743 auto workbook = dom.children[0]; 744 if (workbook.name != "workbook" && workbook.name != "s:workbook") 745 return []; 746 747 const sheetName = workbook.name == "workbook" ? "sheets" : "s:sheets"; 748 auto sheetsRng = workbook.children.filter!(c => c.name == sheetName); 749 if (sheetsRng.empty) 750 return []; 751 752 return sheetsRng 753 .front 754 .children 755 .map!( 756 // TODO: optimize by using indexOf or find 757 s => SheetNameId( 758 s.attributes.filter!(a => a.name == "name").front.value 759 .decodeXML(), 760 s.attributes.filter!(a => a.name == "sheetId").front.value 761 .to!int(), 762 s.attributes.filter!(a => a.name == "r:id").front.value 763 )) 764 .array; 765 } 766 767 /// Returns: Sheets as an eagerly evaluated and internally cached array. 768 auto sheets() { 769 if (_sheets is null) 770 _sheets = bySheet.array; 771 return _sheets; 772 } 773 774 /// Returns: Lazy range over sheets. 775 @SILignore 776 auto bySheet() @safe { 777 return sheetNameIds.map!((const scope SheetNameId sheetNameId) { 778 return extractSheet(_za, relationships, filename, sheetNameId.rid, 779 sheetNameId.name); 780 }); 781 } 782 783 /// Get (and cache) DOM. 784 @SILignore 785 private DOMEntity!(string) workbookDOM() @safe /* TODO: pure */ { 786 import dxml.parser : Config, SkipComments, SkipPI, SplitEmpty, 787 ThrowOnEntityRef; 788 auto ent = workbookXMLPath in _za.directory; 789 // TODO: use enforce(ent ! is null); instead? 790 if (ent is null) 791 return typeof(return).init; 792 if (_wbDOM == _wbDOM.init) { 793 auto est = _za.expandTrusted(*ent).convertToString(); 794 static if (tme) 795 auto sw = StopWatch(AutoStart.yes); 796 _wbDOM = est.parseDOM!(Config( 797 SkipComments 798 .no, // TODO: change to SkipComments.yes and validate 799 SkipPI.no, // TODO: change to SkipPI.yes and validate 800 SplitEmpty.no, // default is ok 801 ThrowOnEntityRef.yes 802 ))(); // default is ok 803 enforce( 804 _wbDOM.children.length == 1, 805 "Expected a single DOM child but got " 806 ~ _wbDOM.children.length.to!string 807 ); 808 static if (tme) 809 writeln("parseDOM length:", est.length, " took: ", sw.peek()); 810 } 811 812 return _wbDOM; 813 } 814 815 @SILignore 816 private RelationshipsById relationships() @safe /* TODO: pure */ { 817 if (_rels is null) 818 _rels = parseRelationships(_za.directory[relsXMLPath]); 819 return _rels; 820 } 821 822 @SILignore 823 private RelationshipsById parseRelationships(ArchiveMember am) @safe { 824 auto est = _za.expandTrusted(am).convertToString(); 825 // import std.digest : digest; 826 // import std.digest.md : MD5; 827 // writeln("am.name:", am.name, " md5:", est.digest!MD5); 828 auto dom = est.parseDOM(); 829 enforce( 830 dom.children.length == 1, 831 "Expected a single DOM child but got " 832 ~ dom.children.length.to!string 833 ); 834 835 auto rel = dom.children[0]; 836 enforce( 837 rel.name == "Relationships", 838 "Expected rel.name to be \"Relationships\" but was " ~ rel.name 839 ); 840 841 typeof(return) ret; 842 static if (is(typeof(ret.reserve(size_t.init)) == void)) { 843 /* Use reserve() when AA gets it or `RelationshipsById` is a custom hash 844 * map. */ 845 ret.reserve(rel.children.length); 846 } 847 848 foreach (ref r; rel.children.filter!(c => c.name == "Relationship")) { 849 Relationships tmp; 850 tmp.id = r.attributes.filter!(a => a.name == "Id").front.value; 851 tmp.file = 852 r.attributes.filter!(a => a.name == "Target").front.value; 853 ret[tmp.id] = tmp; 854 } 855 856 enforce(!ret.empty); 857 return ret; 858 } 859 860 @SILignore 861 private string[] sharedEntries() @safe /* TODO: pure */ { 862 if (_sharedEntries is null) 863 if (ArchiveMember* amPtr = sharedStringXMLPath in _za.directory) 864 _sharedEntries = readSharedEntries(_za, *amPtr); 865 return _sharedEntries; 866 } 867 868 @SILignore 869 const string filename; 870 871 /* TODO: remove when https://github.com/libmir/mir-core/pull/79 */ 872 @SILignore 873 private inout(ZipArchive) _za() inout @safe pure nothrow @nogc { 874 return __za; 875 } 876 877 @SILignore 878 private ZipArchive __za; 879 880 @SILignore 881 private DOMEntity!string _wbDOM; ///< Workbook. 882 883 @SILignore 884 private RelationshipsById _rels; 885 886 @SILignore 887 private string[] _sharedEntries; 888 889 @SILignore 890 private Sheet[] _sheets; 891 } 892 893 version(mir_test) 894 @safe 895 unittest { 896 const sheets = Workbook.fromFile("test/data/50xP_sheet1.xlsx").sheets(); 897 assert(sheets.length == 1); 898 assert(sheets[0].cells.length == 6008); 899 // writeln(sheets[0]); 900 // TODO: that cells are of time Data or DateTime 901 } 902 903 /// benchmark reading of "50xP_sheet1.xlsx" 904 version(mir_benchmark) 905 @safe 906 unittest { 907 const path = "test/data/50xP_sheet1.xlsx"; 908 import std.meta : AliasSeq; 909 void use_sheetNamesAndreadSheet() @trusted { 910 foreach (const _, const ref s; Workbook.fromFile(path).sheets) { 911 } 912 } 913 914 size_t use_bySheet() @trusted { 915 auto file = Workbook.fromFile(path); 916 typeof(return) i = 0; 917 foreach (ref sheet; file.bySheet) { 918 i++; 919 } 920 921 return i; 922 } 923 924 alias funs = AliasSeq!(/* use_sheetNamesAndreadSheet, */ 925 use_bySheet); 926 auto results = benchmarkSum!(funs)(runCount); 927 foreach (const i, fun; funs) { 928 writeln(fun.stringof[0 .. $ - 2], "(\"", path, "\") took ", 929 results[i] / runCount); 930 } 931 } 932 933 /// Sheet name, id and rid. 934 struct SheetNameId { 935 string name; 936 int id; 937 string rid; 938 } 939 940 /// Strip BOM and convert ubyte[] to a string. 941 string convertToString(inout(ubyte)[] d) @trusted { 942 import std.encoding : getBOM, BOM, transcode; 943 const b = getBOM(d); 944 switch (b.schema) { 945 case BOM.none: 946 return cast(string) d; // TODO: remove this cast 947 case BOM.utf8: 948 return cast(string) (d[3 .. $]); // TODO: remove this cast 949 case BOM.utf16be: 950 case BOM.utf16le: 951 case BOM.utf32be: 952 case BOM.utf32le: 953 goto default; 954 default: 955 string ret; 956 transcode(d, ret); 957 return ret; 958 } 959 } 960 961 version(mir_test) 962 @safe 963 unittest { 964 auto r = Workbook.fromFile("test/data/multitable.xlsx").sheetNameIds(); 965 assert(r 966 == [SheetNameId("wb1", 1, "rId2"), SheetNameId("wb2", 2, "rId3"), 967 SheetNameId("Sheet3", 3, "rId4")]); 968 } 969 970 version(mir_test) 971 @safe 972 unittest { 973 auto r = Workbook.fromFile("test/data/sheetnames.xlsx").sheetNameIds(); 974 assert(r == [SheetNameId("A & B ;", 1, "rId2")]); 975 } 976 977 /// Read sheet named `sheetName` from `filename`. 978 Sheet readSheet(in string filename, in string sheetName) @trusted { 979 return Workbook.fromFile(filename).sheets().filter!(sheet => sheet.name == sheetName).front; 980 } 981 982 string eatXlPrefix(scope return string fn) @safe pure nothrow @nogc { 983 static immutable xlPrefixes = ["xl//", "/xl/"]; 984 foreach (const p; xlPrefixes) { 985 // TODO: use fn.skipOver("xl//", "/xl/") when it’s nothrow @nogc 986 if (fn.startsWith(p)) { 987 return fn[p.length .. $]; 988 } 989 } 990 991 return fn; 992 } 993 994 private 995 Sheet extractSheet(ZipArchive za, in RelationshipsById rels, in string filename, 996 in string rid, in string sheetName) @trusted { 997 string[] ss; /* shared strings (table) */ 998 if (ArchiveMember* amPtr = sharedStringXMLPath in za.directory) 999 ss = readSharedEntries( 1000 za, *amPtr); // TODO: cache this into File.sharedStrings 1001 1002 const Relationships* sheetRel = rid 1003 in rels; // TODO: move this calculation to caller and pass Relationships as rels 1004 enforce(sheetRel !is null, 1005 format("Could not find '%s' in '%s'", rid, filename)); 1006 const fn = "xl/" ~ eatXlPrefix(sheetRel.file); 1007 ArchiveMember* sheet = fn in za.directory; 1008 enforce( 1009 sheet !is null, 1010 format("sheetRel.file orig '%s', fn %s not in [%s]", sheetRel.file, fn, 1011 za.directory.keys()) 1012 ); 1013 1014 SparseCell[] cells1 = readCells(za, *sheet); // hot spot! 1015 SparseCell[] cells = insertValueIntoCell(cells1, ss); 1016 1017 Position maxPos = Position.origin; 1018 foreach (ref c; cells) { 1019 c.position = toPos(c.r); 1020 maxPos = elementMax(maxPos, c.position); 1021 } 1022 1023 const extent = 1024 Extent(RowWidth(maxPos.col + 1), ColumnHeight(maxPos.row + 1)); 1025 1026 // debug writeln("filename:", filename, " maxPos:", maxPos, " extent:", extent); 1027 return Sheet(sheetName, cells, extent); 1028 } 1029 1030 string[] readSharedEntries(ZipArchive za, ArchiveMember am) @safe { 1031 auto dom = za.expandTrusted(am).convertToString().parseDOM(); // TODO: cache 1032 if (dom.type != EntityType.elementStart) 1033 return typeof(return).init; 1034 assert(dom.children.length == 1); 1035 1036 auto sst = dom.children[0]; 1037 assert(sst.name == "sst"); 1038 1039 if (sst.type != EntityType.elementStart || sst.children.empty) 1040 return typeof(return).init; 1041 1042 Appender!(typeof(return)) ret; // TODO: reserve? 1043 foreach (ref si; sst.children.filter!(c => c.name == "si")) { 1044 if (si.type != EntityType.elementStart) 1045 continue; 1046 //ret ~= extractData(si); 1047 string tmp; 1048 foreach (ref tORr; si.children) { 1049 if (tORr.name == "t" && tORr.type == EntityType.elementStart 1050 && !tORr.children.empty) { 1051 //ret ~= DenseValue(convert(tORr.children[0].text)); 1052 ret ~= tORr.children[0].text.decodeXML; 1053 } else if (tORr.name == "r") { 1054 foreach (ref r; tORr.children.filter!(r => r.name == "t")) { 1055 if (r.type == EntityType.elementStart 1056 && !r.children.empty) { 1057 tmp ~= r.children[0].text.decodeXML; 1058 } 1059 } 1060 } else { 1061 //ret ~= DenseValue.init; 1062 ret ~= ""; 1063 } 1064 } 1065 1066 if (!tmp.empty) { 1067 //ret ~= DenseValue(convert(tmp)); 1068 ret ~= tmp.decodeXML; 1069 } 1070 } 1071 1072 return ret.data; 1073 } 1074 1075 string extractData(DOMEntity!string si) @safe { 1076 string tmp; 1077 foreach (ref tORr; si.children) { 1078 if (tORr.name == "t") { 1079 if (!tORr.attributes.filter!(a => a.name == "xml:space").empty) { 1080 return ""; 1081 } else if (tORr.type == EntityType.elementStart 1082 && !tORr.children.empty) { 1083 return tORr.children[0].text; 1084 } else { 1085 return ""; 1086 } 1087 } else if (tORr.name == "r") { 1088 foreach (ref r; tORr.children.filter!(r => r.name == "t")) { 1089 tmp ~= r.children[0].text; 1090 } 1091 } 1092 } 1093 1094 if (!tmp.empty) { 1095 return tmp; 1096 } 1097 1098 assert(false); 1099 } 1100 1101 private bool canConvertToLong(in string s) @safe pure nothrow @nogc { 1102 import std.utf : byChar; 1103 import std.ascii : isDigit; 1104 if (s.empty) 1105 return false; 1106 return s.byChar.all!isDigit(); 1107 } 1108 1109 version(ctRegex_test) 1110 version(unittest) { 1111 import std.regex : ctRegex, matchAll; 1112 private static immutable rs = r"[\+-]{0,1}[0-9][0-9]*\.[0-9]*"; 1113 private static immutable rgx = ctRegex!rs; 1114 private bool canConvertToDoubleOld(in string s) @safe { 1115 auto cap = matchAll(s, rgx); 1116 return cap.empty || cap.front.hit != s ? false : true; 1117 } 1118 } 1119 1120 private bool canConvertToDouble(string s) pure @safe nothrow @nogc { 1121 if (s.startsWith('+', '-')) { 1122 s = s[1 .. $]; 1123 } 1124 1125 if (s.empty) { 1126 return false; 1127 } 1128 1129 if (s[0] < '0' || s[0] > '9') { // at least one in [0-9] 1130 return false; 1131 } 1132 1133 s = s[1 .. $]; 1134 1135 if (s.empty) { 1136 return true; 1137 } 1138 1139 while (!s.empty && s[0] >= '0' && s[0] <= '9') { 1140 s = s[1 .. $]; 1141 } 1142 1143 if (s.empty) { 1144 return true; 1145 } 1146 1147 if (s[0] != '.') { 1148 return false; 1149 } 1150 1151 s = s[1 .. $]; 1152 if (s.empty) { 1153 return true; 1154 } 1155 1156 while (!s.empty && s[0] >= '0' && s[0] <= '9') { 1157 s = s[1 .. $]; 1158 } 1159 1160 return s.empty; 1161 } 1162 1163 version(mir_test) 1164 @safe 1165 unittest { 1166 static struct Test { 1167 string tt; 1168 bool rslt; 1169 } 1170 1171 auto tests = [ 1172 Test("-", false), 1173 Test("0.0", true), 1174 Test("-0.", true), 1175 Test("-0.0", true), 1176 Test("-0.a", false), 1177 Test("-0.0", true), 1178 Test("-1100.0", true) 1179 ]; 1180 foreach (const t; tests) { 1181 version(ctRegex_test) 1182 assert( 1183 canConvertToDouble(t.tt) == canConvertToDoubleOld(t.tt), 1184 format("%s %s %s %s", t.tt, canConvertToDouble(t.tt), 1185 canConvertToDoubleOld(t.tt), t.rslt) 1186 ); 1187 assert(canConvertToDouble(t.tt) == t.rslt, 1188 format("%s %s %s", t.tt, canConvertToDouble(t.tt), t.rslt)); 1189 } 1190 } 1191 1192 SparseCell[] readCells(ZipArchive za, ArchiveMember am) @safe { 1193 auto dom = 1194 za.expandTrusted(am).convertToString().parseDOM(); // TODO: cache? 1195 assert(dom.children.length == 1); 1196 1197 auto ws = dom.children[0]; 1198 if (ws.name != "worksheet") 1199 return typeof(return).init; 1200 1201 auto sdRng = ws.children.filter!(c => c.name == "sheetData"); 1202 assert(!sdRng.empty); 1203 1204 if (sdRng.front.type != EntityType.elementStart) 1205 return typeof(return).init; 1206 1207 auto rows = sdRng.front.children.filter!(r => r.name == "row"); 1208 1209 Appender!(typeof(return)) ret; // TODO: reserve()? 1210 foreach (ref row; rows) { 1211 if (row.type != EntityType.elementStart || row.children.empty) { 1212 continue; 1213 } 1214 1215 foreach (ref c; row.children.filter!(r => r.name == "c")) { 1216 SparseCell tmp; 1217 tmp.row = RowOffset(row.attributes.filter!(a => a.name == "r").front 1218 .value.to!(typeof(RowOffset.value))); 1219 tmp.r = c.attributes.filter!(a => a.name == "r").front.value; 1220 auto t = c.attributes.filter!(a => a.name == "t"); 1221 if (t.empty) { 1222 // we assume that no t attribute means direct number 1223 //writefln("Found a strange empty cell \n%s", c); 1224 } else { 1225 tmp.t = t.front.value; 1226 } 1227 1228 if (tmp.t == "s" || tmp.t == "n") { 1229 if (c.type == EntityType.elementStart) { 1230 auto v = c.children.filter!(c => c.name == "v"); 1231 //enforce(!v.empty, format("r %s", tmp.row)); 1232 if (!v.empty && v.front.type == EntityType.elementStart 1233 && !v.front.children.empty) { 1234 tmp.v = v.front.children[0].text; 1235 } else { 1236 tmp.v = ""; 1237 } 1238 } 1239 } else if (tmp.t == "inlineStr") { 1240 auto is_ = c.children.filter!(c => c.name == "is"); 1241 tmp.v = extractData(is_.front); 1242 } else if (c.type == EntityType.elementStart) { 1243 auto v = c.children.filter!(c => c.name == "v"); 1244 if (!v.empty && v.front.type == EntityType.elementStart 1245 && !v.front.children.empty) { 1246 tmp.v = v.front.children[0].text; 1247 } 1248 } 1249 1250 if (c.type == EntityType.elementStart) { 1251 auto f = c.children.filter!(c => c.name == "f"); 1252 if (!f.empty && f.front.type == EntityType.elementStart) { 1253 tmp.formula = f.front.children[0].text; 1254 } 1255 } 1256 1257 ret ~= tmp; 1258 } 1259 } 1260 1261 return ret.data; // TODO: assumeUnique? 1262 } 1263 1264 /** 1265 * Param: `ss` is the shared string (table) 1266 */ 1267 SparseCell[] insertValueIntoCell(SparseCell[] cells, 1268 in string[] ss) @trusted /* TODO: pure */ { 1269 immutable excepted = ["f", /* formula */ 1270 "n", /* number */ 1271 "s", /* string? */ 1272 "d", /* date */ 1273 "b", /* boolean */ 1274 "e", /* string */ 1275 "str", /* string */ 1276 "inlineStr" /* inline string */ 1277 ]; // TODO: what are these? 1278 immutable same = ["n", "e", "str", "inlineStr"]; // TODO: what are these? 1279 foreach (ref SparseCell c; cells) { 1280 // debug writeln("c.t:", c.t, " c.v:", c.v); 1281 assert(excepted.canFind(c.t) || c.t.empty, 1282 format("'%s' not in [%s]", c.t, excepted)); 1283 if (c.t.empty) { 1284 c.xmlValue = c.v.decodeXML; 1285 } else if (same.canFind(c.t)) { 1286 c.xmlValue = c.v.decodeXML; 1287 } else if (c.t == "b") { 1288 c.xmlValue = c.v.decodeXML; 1289 } else if (!c.v.empty) { 1290 c.xmlValue = ss[c.v.to!size_t]; /* shared string table? */ 1291 } 1292 1293 switch (c.t) { 1294 case "b": // boolean 1295 if (c.xmlValue == "0") 1296 c.value = false; 1297 else if (c.xmlValue == "1") 1298 c.value = true; 1299 else 1300 c.value = c.xmlValue; 1301 break; 1302 case "n": // number 1303 if (c.v.canFind(".")) 1304 c.value = c.v.to!double; 1305 else 1306 c.value = c.v.to!long; 1307 break; 1308 case "d": // date 1309 // TODO: c.value = c.xmlValue.convertTo!DateTime; 1310 c.value = c.xmlValue; 1311 break; 1312 default: 1313 c.value = c.xmlValue; 1314 break; 1315 } 1316 } 1317 1318 return cells; 1319 } 1320 1321 Position toPos(in string s) @safe pure { 1322 import std.string : indexOfAny; 1323 import std.math : pow; 1324 ptrdiff_t fn = s.indexOfAny("0123456789"); 1325 enforce(fn != -1, s); 1326 RowOffset row = to!RowOffset(to!int(s[fn .. $]) - 1); 1327 ColumnOffset col = ColumnOffset(0); 1328 string colS = s[0 .. fn]; 1329 foreach (const idx, char c; colS) { 1330 col = col * 26 + (c - 'A' + 1); 1331 } 1332 1333 return Position(row, ColumnOffset(col - 1)); 1334 } 1335 1336 version(mir_test) 1337 @safe 1338 pure unittest { 1339 assert(toPos("A1").col == 0); 1340 assert(toPos("Z1").col == 25); 1341 assert(toPos("AA1").col == 26); 1342 } 1343 1344 Position elementMax(Position a, Position b) @safe pure nothrow @nogc { 1345 import std.algorithm.comparison : max; 1346 return Position(max(a.row, b.row), max(a.col, b.col)); 1347 } 1348 1349 version(mir_test) 1350 @safe 1351 unittest { 1352 import std.math : isClose; 1353 auto r = readSheet("test/data/multitable.xlsx", "wb1"); 1354 { 1355 const e = r.denseTable[RowOffset(12),ColumnOffset(5)]; 1356 assert(isClose(e.xmlValue.to!double(), 26.74)); 1357 } 1358 { 1359 const e = r.denseTable[RowOffset(13), ColumnOffset(5)]; 1360 assert(isClose(e.xmlValue.to!double(), -26.74)); 1361 } 1362 } 1363 1364 version(mir_test) 1365 @safe 1366 unittest { 1367 auto s = readSheet("test/data/multitable.xlsx", "wb1"); 1368 const expectedCells = [ 1369 SparseCell(RowOffset(3), "s", "D3", "0", "", "a", 1370 Position(RowOffset(2), ColumnOffset(3))), 1371 SparseCell(RowOffset(3), "s", "E3", "1", "", "b", 1372 Position(RowOffset(2), ColumnOffset(4))), 1373 SparseCell(RowOffset(4), "s", "D4", "2", "", "1", 1374 Position(RowOffset(3), ColumnOffset(3))), 1375 SparseCell(RowOffset(4), "s", "E4", "3", "", "\"one\"", 1376 Position(RowOffset(3), ColumnOffset(4))), 1377 SparseCell(RowOffset(5), "s", "D5", "4", "", "2", 1378 Position(RowOffset(4), ColumnOffset(3))), 1379 SparseCell(RowOffset(5), "s", "E5", "5", "", "\"two\"", 1380 Position(RowOffset(4), ColumnOffset(4))), 1381 SparseCell(RowOffset(6), "s", "D6", "6", "", "3", 1382 Position(RowOffset(5), ColumnOffset(3))), 1383 SparseCell(RowOffset(6), "s", "E6", "7", "", "\"three\"", 1384 Position(RowOffset(5), ColumnOffset(4))), 1385 SparseCell(RowOffset(7), "", "B7", "", "", "", 1386 Position(RowOffset(6), ColumnOffset(1))), 1387 SparseCell(RowOffset(7), "s", "F7", "1", "", "b", 1388 Position(RowOffset(6), ColumnOffset(5))), 1389 SparseCell(RowOffset(7), "s", "G7", "0", "", "a", 1390 Position(RowOffset(6), ColumnOffset(6))), 1391 SparseCell(RowOffset(8), "n", "C8", "0.504409722222222", "", 1392 "0.504409722222222", Position(RowOffset(7), ColumnOffset(2))), 1393 SparseCell(RowOffset(8), "s", "F8", "3", "", "\"one\"", 1394 Position(RowOffset(7), ColumnOffset(5))), 1395 SparseCell(RowOffset(8), "s", "G8", "2", "", "1", 1396 Position(RowOffset(7), ColumnOffset(6))), 1397 SparseCell(RowOffset(9), "s", "F9", "5", "", "\"two\"", 1398 Position(RowOffset(8), ColumnOffset(5))), 1399 SparseCell(RowOffset(9), "s", "G9", "4", "", "2", 1400 Position(RowOffset(8), ColumnOffset(6))), 1401 SparseCell(RowOffset(10), "s", "F10", "7", "", "\"three\"", 1402 Position(RowOffset(9), ColumnOffset(5))), 1403 SparseCell(RowOffset(10), "s", "G10", "6", "", "3", Position(RowOffset(9), ColumnOffset(6))), 1404 SparseCell(RowOffset(11), "s", "AC11", "8", "", "Foo", Position(RowOffset(10), ColumnOffset(28))), 1405 SparseCell(RowOffset(12), "s", "B12", "9", "", "Hello World", Position(RowOffset(11), ColumnOffset(1))), 1406 SparseCell(RowOffset(13), "n", "E13", "13.37", "", "13.37", Position(RowOffset(12), ColumnOffset(4))), 1407 SparseCell(RowOffset(13), "n", "F13", "26.74", "E13*2", "26.74", Position(RowOffset(12), ColumnOffset(5))), 1408 SparseCell(RowOffset(14), "n", "F14", "-26.74", "-E13*2", "-26.74", Position(RowOffset(13), ColumnOffset(5))), 1409 SparseCell(RowOffset(16), "n", "B16", "1", "", "1", Position(RowOffset(15), ColumnOffset(1))), 1410 SparseCell(RowOffset(16), "n", "C16", "2", "", "2", Position(RowOffset(15), ColumnOffset(2))), 1411 SparseCell(RowOffset(16), "n", "D16", "3", "", "3", Position(RowOffset(15), ColumnOffset(3))), 1412 SparseCell(RowOffset(16), "n", "E16", "4", "", "4", Position(RowOffset(15), ColumnOffset(4))), 1413 SparseCell(RowOffset(16), "n", "F16", "5", "", "5", Position(RowOffset(15), ColumnOffset(5))) 1414 ]; 1415 foreach (const i, const ref cell; s.cells) { 1416 // compare all but last field for now as that’s subject to change 1417 assert( 1418 cell.tupleof[0 .. $ - 1] == expectedCells[i].tupleof[0 .. $ - 1] 1419 ); 1420 } 1421 1422 assert(s.extent.width == 29); 1423 assert(s.extent.height == 16); 1424 assert(s.denseTable.cells.length == 29*16); 1425 } 1426 1427 version(mir_test) 1428 @safe 1429 unittest { 1430 auto s = readSheet("test/data/multitable.xlsx", "wb1"); 1431 const expected = [1, 2, 3, 4, 5]; 1432 auto r = s.getRow(RowOffset(15), ColumnOffset(1), ColumnOffset(6)).map!(_ => _.value); 1433 assert(equal(r, expected)); 1434 assert(equal(s.getRow(RowOffset(15), ColumnOffset(1), ColumnOffset(6)), 1435 s.getRow(RowOffset(15), ColumnOffset(1), ColumnOffset(6)))); 1436 } 1437 1438 version(mir_test) 1439 @safe 1440 unittest { 1441 auto s = readSheet("test/data/multitable.xlsx", "wb2"); 1442 auto r = s.getColumn(ColumnOffset(1), RowOffset(1), RowOffset(6)); 1443 auto expected = [Date(2019, 5, 01), Date(2016, 12, 27), Date(1976, 7, 23), 1444 Date(1986, 7, 2), Date(2038, 1, 19)]; 1445 version(none) assert(equal(r, expected)); // TODO: enable when Value Date(Time) decoding bugs have been fixed 1446 } 1447 1448 version(mir_test) 1449 @safe 1450 unittest { 1451 auto s = readSheet("test/data/multitable.xlsx", "Sheet3"); 1452 assert(s.denseTable[RowOffset(0), ColumnOffset(0)].xmlValue.to!long(), 1453 format("%s", s.denseTable[RowOffset(0), ColumnOffset(0)].xmlValue)); 1454 //assert(s.denseTable[RowOffset(0), ColumnOffset(0)].canConvertTo(CellType.bool_)); 1455 } 1456 1457 version(mir_test) 1458 @system 1459 unittest { 1460 import std.file : dirEntries, SpanMode; 1461 import std.algorithm.searching : endsWith; 1462 size_t totalSheetCount; 1463 size_t totalCellCount; 1464 1465 /* Reading the file row_col_format16.xlsx causes out of memory because one of its cells position address strings is 1466 * mapped to the column index 16384. 1467 */ 1468 foreach (const de; 1469 dirEntries("test/data/xlsx_files/", "*.xlsx", SpanMode.depth) 1470 .filter!(a => (!a.name.endsWith( 1471 "data02.xlsx", "data03.xlsx", "data04.xlsx", 1472 "row_col_format16.xlsx", "row_col_format18.xlsx")))) { 1473 auto sn = Workbook.fromFile(de.name).sheetNameIds; 1474 foreach (const s; sn) { 1475 totalSheetCount += 1; 1476 auto sheet = readSheet(de.name, s.name); 1477 foreach (const cell; sheet.cells) { 1478 totalCellCount += 1; 1479 } 1480 } 1481 } 1482 1483 assert(totalSheetCount == 551); 1484 assert(totalCellCount == 4860); 1485 } 1486 1487 version(mir_test) 1488 @safe 1489 unittest { 1490 auto sheet = readSheet("test/data/testworkbook.xlsx", "ws1"); 1491 1492 assert(sheet.denseTable[RowOffset(2), ColumnOffset(3)].xmlValue == "1337"); 1493 assert(sheet.denseTable[RowOffset(2), ColumnOffset(4)].xmlValue == "hello"); 1494 assert(sheet.denseTable[RowOffset(3), ColumnOffset(4)].xmlValue == "sil"); 1495 assert(sheet.denseTable[RowOffset(4), ColumnOffset(4)].xmlValue == "foo"); 1496 1497 auto r1 = sheet.getColumn(ColumnOffset(3), RowOffset(2), RowOffset(5)).map!(_ => _.value); 1498 assert(equal(r1, ["1337", "2", "3"])); 1499 1500 auto r2 = sheet.getColumn(ColumnOffset(4), RowOffset(2), RowOffset(5)).map!(_ => _.value); 1501 assert(equal(r2, ["hello", "sil", "foo"])); 1502 } 1503 1504 version(mir_test) 1505 @safe 1506 unittest { 1507 import std.math : isClose; 1508 auto s = readSheet("test/data/toto.xlsx", "Trades"); 1509 auto r = s.getRow(RowOffset(1), ColumnOffset(0), ColumnOffset(2)).array; 1510 assert(isClose(r[1].value.get!double, 38204642.510000)); 1511 } 1512 1513 version(mir_test) 1514 @safe 1515 unittest { 1516 const sheet = readSheet("test/data/leading_zeros.xlsx", "Sheet1"); 1517 auto a2 = sheet.cells.filter!(c => c.r == "A2"); 1518 assert(!a2.empty); 1519 assert(a2.front.xmlValue == "0012"); 1520 } 1521 1522 version(mir_test) 1523 @safe 1524 unittest { 1525 auto s = readSheet("test/data/datetimes.xlsx", "Sheet1"); 1526 auto r = s.getColumn(ColumnOffset(0), RowOffset(0), RowOffset(2)); 1527 assert(equal(r, [DenseCell(DenseValue(31423), "31423", ""), 1528 DenseCell(DenseValue(31595), "31595", "")])); 1529 auto expected = [DateTime(Date(1986, 1, 11), TimeOfDay.init), 1530 DateTime(Date(1986, 7, 2), TimeOfDay.init)]; 1531 version(none) assert(equal(r, expected)); // TODO: enable when Value Date(Time) decoding bugs have been fixed 1532 } 1533 1534 version(mir_test) { 1535 import std.algorithm.comparison : equal; 1536 } 1537 1538 /** Variant of Phobos `benchmark` that, instead of sum all run times, returns 1539 minimum of all run times as that is a more stable metric. 1540 */ 1541 version(mir_benchmark) 1542 { 1543 private 1544 Duration[funs.length] benchmarkSum(funs...)(uint n) if (funs.length >= 1) { 1545 import std.algorithm.comparison : min; 1546 Duration[funs.length] result; 1547 auto sw = StopWatch(AutoStart.yes); 1548 foreach (const i, fun; funs) { 1549 result[i] = Duration.init; 1550 foreach (const j; 0 .. n) { 1551 sw.reset(); 1552 sw.start(); 1553 fun(); 1554 sw.stop(); 1555 result[i] += sw.peek(); 1556 } 1557 } 1558 return result; 1559 } 1560 private 1561 Duration[funs.length] benchmarkMin(funs...)(uint n) if (funs.length >= 1) { 1562 import std.algorithm.comparison : min; 1563 Duration[funs.length] result; 1564 auto sw = StopWatch(AutoStart.yes); 1565 foreach (const i, fun; funs) { 1566 result[i] = Duration.max; 1567 foreach (const j; 0 .. n) { 1568 sw.reset(); 1569 sw.start(); 1570 fun(); 1571 sw.stop(); 1572 result[i] = min(result[i], sw.peek()); 1573 } 1574 } 1575 return result; 1576 } 1577 }