| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/globals.h" // Needed here to get TARGET_ARCH_X64. | 5 #include "vm/globals.h" // Needed here to get TARGET_ARCH_X64. |
| 6 #if defined(TARGET_ARCH_X64) | 6 #if defined(TARGET_ARCH_X64) |
| 7 | 7 |
| 8 #include "vm/intrinsifier.h" | 8 #include "vm/intrinsifier.h" |
| 9 | 9 |
| 10 #include "vm/assembler.h" | 10 #include "vm/assembler.h" |
| (...skipping 421 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 432 __ movq(R12, Address(RSP, + 1 * kWordSize)); // Index. | 432 __ movq(R12, Address(RSP, + 1 * kWordSize)); // Index. |
| 433 __ testq(R12, Immediate(kSmiTagMask)); | 433 __ testq(R12, Immediate(kSmiTagMask)); |
| 434 __ j(NOT_ZERO, fall_through, Assembler::kNearJump); // Non-smi index. | 434 __ j(NOT_ZERO, fall_through, Assembler::kNearJump); // Non-smi index. |
| 435 // Range check. | 435 // Range check. |
| 436 __ cmpq(R12, FieldAddress(RAX, ByteArray::length_offset())); | 436 __ cmpq(R12, FieldAddress(RAX, ByteArray::length_offset())); |
| 437 // Runtime throws exception. | 437 // Runtime throws exception. |
| 438 __ j(ABOVE_EQUAL, fall_through, Assembler::kNearJump); | 438 __ j(ABOVE_EQUAL, fall_through, Assembler::kNearJump); |
| 439 } | 439 } |
| 440 | 440 |
| 441 | 441 |
| 442 #define TYPED_ARRAY_ALLOCATION(type_name, scale_factor) \ | 442 #define TYPED_ARRAY_ALLOCATION(type_name, cid, max_len, scale_factor) \ |
| 443 Label fall_through; \ | 443 Label fall_through; \ |
| 444 const intptr_t kArrayLengthStackOffset = 1 * kWordSize; \ | 444 const intptr_t kArrayLengthStackOffset = 1 * kWordSize; \ |
| 445 __ movq(RDI, Address(RSP, kArrayLengthStackOffset)); /* Array length. */ \ | 445 __ movq(RDI, Address(RSP, kArrayLengthStackOffset)); /* Array length. */ \ |
| 446 /* Check that length is a positive Smi. */ \ | 446 /* Check that length is a positive Smi. */ \ |
| 447 /* RDI: requested array length argument. */ \ | 447 /* RDI: requested array length argument. */ \ |
| 448 __ testq(RDI, Immediate(kSmiTagSize)); \ | 448 __ testq(RDI, Immediate(kSmiTagSize)); \ |
| 449 __ j(NOT_ZERO, &fall_through); \ | 449 __ j(NOT_ZERO, &fall_through); \ |
| 450 __ cmpq(RDI, Immediate(0)); \ | 450 __ cmpq(RDI, Immediate(0)); \ |
| 451 __ j(LESS, &fall_through); \ | 451 __ j(LESS, &fall_through); \ |
| 452 __ SmiUntag(RDI); \ | 452 __ SmiUntag(RDI); \ |
| 453 /* Check for maximum allowed length. */ \ | 453 /* Check for maximum allowed length. */ \ |
| 454 /* RDI: untagged array length. */ \ | 454 /* RDI: untagged array length. */ \ |
| 455 __ cmpq(RDI, Immediate(type_name::kMaxElements)); \ | 455 __ cmpq(RDI, Immediate(max_len)); \ |
| 456 __ j(GREATER, &fall_through); \ | 456 __ j(GREATER, &fall_through); \ |
| 457 const intptr_t fixed_size = sizeof(Raw##type_name) + kObjectAlignment - 1; \ | 457 const intptr_t fixed_size = sizeof(Raw##type_name) + kObjectAlignment - 1; \ |
| 458 __ leaq(RDI, Address(RDI, scale_factor, fixed_size)); \ | 458 __ leaq(RDI, Address(RDI, scale_factor, fixed_size)); \ |
| 459 __ andq(RDI, Immediate(-kObjectAlignment)); \ | 459 __ andq(RDI, Immediate(-kObjectAlignment)); \ |
| 460 Heap* heap = Isolate::Current()->heap(); \ | 460 Heap* heap = Isolate::Current()->heap(); \ |
| 461 \ | 461 \ |
| 462 __ movq(RAX, Immediate(heap->TopAddress())); \ | 462 __ movq(RAX, Immediate(heap->TopAddress())); \ |
| 463 __ movq(RAX, Address(RAX, 0)); \ | 463 __ movq(RAX, Address(RAX, 0)); \ |
| 464 __ movq(RCX, RAX); \ | 464 __ movq(RCX, RAX); \ |
| 465 \ | 465 \ |
| (...skipping 26 matching lines...) Expand all Loading... |
| 492 __ cmpq(RDI, Immediate(RawObject::SizeTag::kMaxSizeTag)); \ | 492 __ cmpq(RDI, Immediate(RawObject::SizeTag::kMaxSizeTag)); \ |
| 493 __ j(ABOVE, &size_tag_overflow, Assembler::kNearJump); \ | 493 __ j(ABOVE, &size_tag_overflow, Assembler::kNearJump); \ |
| 494 __ shlq(RDI, Immediate(RawObject::kSizeTagBit - kObjectAlignmentLog2)); \ | 494 __ shlq(RDI, Immediate(RawObject::kSizeTagBit - kObjectAlignmentLog2)); \ |
| 495 __ jmp(&done, Assembler::kNearJump); \ | 495 __ jmp(&done, Assembler::kNearJump); \ |
| 496 \ | 496 \ |
| 497 __ Bind(&size_tag_overflow); \ | 497 __ Bind(&size_tag_overflow); \ |
| 498 __ movq(RDI, Immediate(0)); \ | 498 __ movq(RDI, Immediate(0)); \ |
| 499 __ Bind(&done); \ | 499 __ Bind(&done); \ |
| 500 \ | 500 \ |
| 501 /* Get the class index and insert it into the tags. */ \ | 501 /* Get the class index and insert it into the tags. */ \ |
| 502 __ orq(RDI, Immediate(RawObject::ClassIdTag::encode(k##type_name##Cid))); \ | 502 __ orq(RDI, Immediate(RawObject::ClassIdTag::encode(cid))); \ |
| 503 __ movq(FieldAddress(RAX, type_name::tags_offset()), RDI); /* Tags. */ \ | 503 __ movq(FieldAddress(RAX, type_name::tags_offset()), RDI); /* Tags. */ \ |
| 504 } \ | 504 } \ |
| 505 /* Set the length field. */ \ | 505 /* Set the length field. */ \ |
| 506 /* RAX: new object start as a tagged pointer. */ \ | 506 /* RAX: new object start as a tagged pointer. */ \ |
| 507 /* RCX: new object end address. */ \ | 507 /* RCX: new object end address. */ \ |
| 508 __ movq(RDI, Address(RSP, kArrayLengthStackOffset)); /* Array length. */ \ | 508 __ movq(RDI, Address(RSP, kArrayLengthStackOffset)); /* Array length. */ \ |
| 509 __ StoreIntoObjectNoBarrier(RAX, \ | 509 __ StoreIntoObjectNoBarrier(RAX, \ |
| 510 FieldAddress(RAX, type_name::length_offset()), \ | 510 FieldAddress(RAX, type_name::length_offset()), \ |
| 511 RDI); \ | 511 RDI); \ |
| 512 /* Initialize all array elements to 0. */ \ | 512 /* Initialize all array elements to 0. */ \ |
| (...skipping 10 matching lines...) Expand all Loading... |
| 523 __ j(ABOVE_EQUAL, &done, Assembler::kNearJump); \ | 523 __ j(ABOVE_EQUAL, &done, Assembler::kNearJump); \ |
| 524 __ movq(Address(RDI, 0), RBX); \ | 524 __ movq(Address(RDI, 0), RBX); \ |
| 525 __ addq(RDI, Immediate(kWordSize)); \ | 525 __ addq(RDI, Immediate(kWordSize)); \ |
| 526 __ jmp(&init_loop, Assembler::kNearJump); \ | 526 __ jmp(&init_loop, Assembler::kNearJump); \ |
| 527 __ Bind(&done); \ | 527 __ Bind(&done); \ |
| 528 \ | 528 \ |
| 529 __ ret(); \ | 529 __ ret(); \ |
| 530 __ Bind(&fall_through); \ | 530 __ Bind(&fall_through); \ |
| 531 | 531 |
| 532 | 532 |
| 533 bool Intrinsifier::Int8Array_new(Assembler* assembler) { | 533 #define SCALARLIST_ALLOCATOR(clazz, scale) \ |
| 534 TYPED_ARRAY_ALLOCATION(Int8Array, TIMES_1); | 534 bool Intrinsifier::clazz##_new(Assembler* assembler) { \ |
| 535 return false; | 535 TYPED_ARRAY_ALLOCATION(clazz, k##clazz##Cid, clazz::kMaxElements, scale); \ |
| 536 return false; \ |
| 537 } \ |
| 538 bool Intrinsifier::clazz##_factory(Assembler* assembler) { \ |
| 539 TYPED_ARRAY_ALLOCATION(clazz, k##clazz##Cid, clazz::kMaxElements, scale); \ |
| 540 return false; \ |
| 536 } | 541 } |
| 537 | 542 |
| 538 | 543 |
| 539 bool Intrinsifier::Int8Array_factory(Assembler* assembler) { | 544 SCALARLIST_ALLOCATOR(Int8Array, TIMES_1) |
| 540 TYPED_ARRAY_ALLOCATION(Int8Array, TIMES_1); | 545 SCALARLIST_ALLOCATOR(Uint8Array, TIMES_1) |
| 541 return false; | 546 SCALARLIST_ALLOCATOR(Uint8ClampedArray, TIMES_1) |
| 542 } | 547 SCALARLIST_ALLOCATOR(Int16Array, TIMES_2) |
| 543 | 548 SCALARLIST_ALLOCATOR(Uint16Array, TIMES_2) |
| 544 | 549 SCALARLIST_ALLOCATOR(Int32Array, TIMES_4) |
| 545 bool Intrinsifier::Uint8Array_new(Assembler* assembler) { | 550 SCALARLIST_ALLOCATOR(Uint32Array, TIMES_4) |
| 546 TYPED_ARRAY_ALLOCATION(Uint8Array, TIMES_1); | 551 SCALARLIST_ALLOCATOR(Int64Array, TIMES_8) |
| 547 return false; | 552 SCALARLIST_ALLOCATOR(Uint64Array, TIMES_8) |
| 548 } | 553 SCALARLIST_ALLOCATOR(Float32Array, TIMES_4) |
| 549 | 554 SCALARLIST_ALLOCATOR(Float64Array, TIMES_8) |
| 550 | |
| 551 bool Intrinsifier::Uint8Array_factory(Assembler* assembler) { | |
| 552 TYPED_ARRAY_ALLOCATION(Uint8Array, TIMES_1); | |
| 553 return false; | |
| 554 } | |
| 555 | |
| 556 | |
| 557 bool Intrinsifier::Uint8ClampedArray_new(Assembler* assembler) { | |
| 558 TYPED_ARRAY_ALLOCATION(Uint8ClampedArray, TIMES_1); | |
| 559 return false; | |
| 560 } | |
| 561 | |
| 562 | |
| 563 bool Intrinsifier::Uint8ClampedArray_factory(Assembler* assembler) { | |
| 564 TYPED_ARRAY_ALLOCATION(Uint8ClampedArray, TIMES_1); | |
| 565 return false; | |
| 566 } | |
| 567 | |
| 568 | |
| 569 bool Intrinsifier::Int16Array_new(Assembler* assembler) { | |
| 570 TYPED_ARRAY_ALLOCATION(Int16Array, TIMES_2); | |
| 571 return false; | |
| 572 } | |
| 573 | |
| 574 | |
| 575 bool Intrinsifier::Int16Array_factory(Assembler* assembler) { | |
| 576 TYPED_ARRAY_ALLOCATION(Int16Array, TIMES_2); | |
| 577 return false; | |
| 578 } | |
| 579 | |
| 580 | |
| 581 bool Intrinsifier::Uint16Array_new(Assembler* assembler) { | |
| 582 TYPED_ARRAY_ALLOCATION(Uint16Array, TIMES_2); | |
| 583 return false; | |
| 584 } | |
| 585 | |
| 586 | |
| 587 bool Intrinsifier::Uint16Array_factory(Assembler* assembler) { | |
| 588 TYPED_ARRAY_ALLOCATION(Uint16Array, TIMES_2); | |
| 589 return false; | |
| 590 } | |
| 591 | |
| 592 | |
| 593 bool Intrinsifier::Int32Array_new(Assembler* assembler) { | |
| 594 TYPED_ARRAY_ALLOCATION(Int32Array, TIMES_4); | |
| 595 return false; | |
| 596 } | |
| 597 | |
| 598 | |
| 599 bool Intrinsifier::Int32Array_factory(Assembler* assembler) { | |
| 600 TYPED_ARRAY_ALLOCATION(Int32Array, TIMES_4); | |
| 601 return false; | |
| 602 } | |
| 603 | |
| 604 | |
| 605 bool Intrinsifier::Uint32Array_new(Assembler* assembler) { | |
| 606 TYPED_ARRAY_ALLOCATION(Uint32Array, TIMES_4); | |
| 607 return false; | |
| 608 } | |
| 609 | |
| 610 | |
| 611 bool Intrinsifier::Uint32Array_factory(Assembler* assembler) { | |
| 612 TYPED_ARRAY_ALLOCATION(Uint32Array, TIMES_4); | |
| 613 return false; | |
| 614 } | |
| 615 | 555 |
| 616 | 556 |
| 617 bool Intrinsifier::Int64Array_getIndexed(Assembler* assembler) { | 557 bool Intrinsifier::Int64Array_getIndexed(Assembler* assembler) { |
| 618 Label fall_through; | 558 Label fall_through; |
| 619 TestByteArrayGetIndex(assembler, &fall_through); | 559 TestByteArrayGetIndex(assembler, &fall_through); |
| 620 // R12: index as Smi. | 560 // R12: index as Smi. |
| 621 // RAX: array. | 561 // RAX: array. |
| 622 __ movq(RAX, FieldAddress(RAX, | 562 __ movq(RAX, FieldAddress(RAX, |
| 623 R12, | 563 R12, |
| 624 TIMES_4, | 564 TIMES_4, |
| 625 Int64Array::data_offset())); | 565 Int64Array::data_offset())); |
| 626 // Copy RAX into R12. | 566 // Copy RAX into R12. |
| 627 // We destroy R12 while testing if RAX can fit inside a Smi. | 567 // We destroy R12 while testing if RAX can fit inside a Smi. |
| 628 __ movq(R12, RAX); | 568 __ movq(R12, RAX); |
| 629 // Verify that the signed value in RAX can fit inside a Smi. | 569 // Verify that the signed value in RAX can fit inside a Smi. |
| 630 __ shlq(R12, Immediate(0x1)); | 570 __ shlq(R12, Immediate(0x1)); |
| 631 // Jump to fall_through if it can not. | 571 // Jump to fall_through if it can not. |
| 632 __ j(OVERFLOW, &fall_through, Assembler::kNearJump); | 572 __ j(OVERFLOW, &fall_through, Assembler::kNearJump); |
| 633 __ SmiTag(RAX); | 573 __ SmiTag(RAX); |
| 634 __ ret(); | 574 __ ret(); |
| 635 __ Bind(&fall_through); | 575 __ Bind(&fall_through); |
| 636 return false; | 576 return false; |
| 637 } | 577 } |
| 638 | 578 |
| 639 | 579 |
| 640 bool Intrinsifier::Int64Array_new(Assembler* assembler) { | |
| 641 TYPED_ARRAY_ALLOCATION(Int64Array, TIMES_8); | |
| 642 return false; | |
| 643 } | |
| 644 | |
| 645 | |
| 646 bool Intrinsifier::Int64Array_factory(Assembler* assembler) { | |
| 647 TYPED_ARRAY_ALLOCATION(Int64Array, TIMES_8); | |
| 648 return false; | |
| 649 } | |
| 650 | |
| 651 | |
| 652 bool Intrinsifier::Uint64Array_getIndexed(Assembler* assembler) { | 580 bool Intrinsifier::Uint64Array_getIndexed(Assembler* assembler) { |
| 653 Label fall_through; | 581 Label fall_through; |
| 654 TestByteArrayGetIndex(assembler, &fall_through); | 582 TestByteArrayGetIndex(assembler, &fall_through); |
| 655 // R12: index as Smi. | 583 // R12: index as Smi. |
| 656 // RAX: array. | 584 // RAX: array. |
| 657 __ movq(RAX, FieldAddress(RAX, | 585 __ movq(RAX, FieldAddress(RAX, |
| 658 R12, | 586 R12, |
| 659 TIMES_4, | 587 TIMES_4, |
| 660 Uint64Array::data_offset())); | 588 Uint64Array::data_offset())); |
| 661 // Copy RAX into R12. | 589 // Copy RAX into R12. |
| 662 // We destroy R12 while testing if RAX can fit inside a Smi. | 590 // We destroy R12 while testing if RAX can fit inside a Smi. |
| 663 __ movq(R12, RAX); | 591 __ movq(R12, RAX); |
| 664 // Verify that the unsigned value in RAX can be stored in a Smi. | 592 // Verify that the unsigned value in RAX can be stored in a Smi. |
| 665 __ shrq(R12, Immediate(kSmiBits)); | 593 __ shrq(R12, Immediate(kSmiBits)); |
| 666 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Won't fit Smi. | 594 __ j(NOT_ZERO, &fall_through, Assembler::kNearJump); // Won't fit Smi. |
| 667 __ SmiTag(RAX); | 595 __ SmiTag(RAX); |
| 668 __ ret(); | 596 __ ret(); |
| 669 __ Bind(&fall_through); | 597 __ Bind(&fall_through); |
| 670 return false; | 598 return false; |
| 671 } | 599 } |
| 672 | 600 |
| 673 | 601 |
| 674 bool Intrinsifier::Uint64Array_new(Assembler* assembler) { | 602 // Gets the length of a TypedData. |
| 675 TYPED_ARRAY_ALLOCATION(Uint64Array, TIMES_8); | 603 bool Intrinsifier::TypedData_getLength(Assembler* assembler) { |
| 676 return false; | 604 __ movq(RAX, Address(RSP, + 1 * kWordSize)); |
| 605 __ movq(RAX, FieldAddress(RAX, TypedData::length_offset())); |
| 606 __ ret(); |
| 607 // Generate enough code to satisfy patchability constraint. |
| 608 intptr_t offset = __ CodeSize(); |
| 609 __ nop(JumpPattern::InstructionLength() - offset); |
| 610 return true; |
| 677 } | 611 } |
| 678 | 612 |
| 679 | 613 |
| 680 bool Intrinsifier::Uint64Array_factory(Assembler* assembler) { | 614 static ScaleFactor GetScaleFactor(intptr_t size) { |
| 681 TYPED_ARRAY_ALLOCATION(Uint64Array, TIMES_8); | 615 switch (size) { |
| 682 return false; | 616 case 1: return TIMES_1; |
| 683 } | 617 case 2: return TIMES_2; |
| 618 case 4: return TIMES_4; |
| 619 case 8: return TIMES_8; |
| 620 } |
| 621 UNREACHABLE(); |
| 622 return static_cast<ScaleFactor>(0); |
| 623 }; |
| 684 | 624 |
| 685 | 625 |
| 686 bool Intrinsifier::Float32Array_new(Assembler* assembler) { | 626 #define TYPEDDATA_ALLOCATOR(clazz) \ |
| 687 TYPED_ARRAY_ALLOCATION(Float32Array, TIMES_4); | 627 bool Intrinsifier::TypedData_##clazz##_new(Assembler* assembler) { \ |
| 688 return false; | 628 intptr_t size = TypedData::ElementSizeInBytes(kTypedData##clazz##Cid); \ |
| 629 intptr_t max_len = kSmiMax / size; \ |
| 630 ScaleFactor scale = GetScaleFactor(size); \ |
| 631 TYPED_ARRAY_ALLOCATION(TypedData, kTypedData##clazz##Cid, max_len, scale); \ |
| 632 return false; \ |
| 633 } \ |
| 634 bool Intrinsifier::TypedData_##clazz##_factory(Assembler* assembler) { \ |
| 635 intptr_t size = TypedData::ElementSizeInBytes(kTypedData##clazz##Cid); \ |
| 636 intptr_t max_len = kSmiMax / size; \ |
| 637 ScaleFactor scale = GetScaleFactor(size); \ |
| 638 TYPED_ARRAY_ALLOCATION(TypedData, kTypedData##clazz##Cid, max_len, scale); \ |
| 639 return false; \ |
| 689 } | 640 } |
| 690 | 641 CLASS_LIST_TYPED_DATA(TYPEDDATA_ALLOCATOR) |
| 691 | 642 #undef TYPEDDATA_ALLOCATOR |
| 692 bool Intrinsifier::Float32Array_factory(Assembler* assembler) { | |
| 693 TYPED_ARRAY_ALLOCATION(Float32Array, TIMES_4); | |
| 694 return false; | |
| 695 } | |
| 696 | |
| 697 | |
| 698 bool Intrinsifier::Float64Array_new(Assembler* assembler) { | |
| 699 TYPED_ARRAY_ALLOCATION(Float64Array, TIMES_8); | |
| 700 return false; | |
| 701 } | |
| 702 | |
| 703 | |
| 704 bool Intrinsifier::Float64Array_factory(Assembler* assembler) { | |
| 705 TYPED_ARRAY_ALLOCATION(Float64Array, TIMES_8); | |
| 706 return false; | |
| 707 } | |
| 708 | 643 |
| 709 | 644 |
| 710 // Tests if two top most arguments are smis, jumps to label not_smi if not. | 645 // Tests if two top most arguments are smis, jumps to label not_smi if not. |
| 711 // Topmost argument is in RAX. | 646 // Topmost argument is in RAX. |
| 712 static void TestBothArgumentsSmis(Assembler* assembler, Label* not_smi) { | 647 static void TestBothArgumentsSmis(Assembler* assembler, Label* not_smi) { |
| 713 __ movq(RAX, Address(RSP, + 1 * kWordSize)); | 648 __ movq(RAX, Address(RSP, + 1 * kWordSize)); |
| 714 __ movq(RCX, Address(RSP, + 2 * kWordSize)); | 649 __ movq(RCX, Address(RSP, + 2 * kWordSize)); |
| 715 __ orq(RCX, RAX); | 650 __ orq(RCX, RAX); |
| 716 __ testq(RCX, Immediate(kSmiTagMask)); | 651 __ testq(RCX, Immediate(kSmiTagMask)); |
| 717 __ j(NOT_ZERO, not_smi, Assembler::kNearJump); | 652 __ j(NOT_ZERO, not_smi, Assembler::kNearJump); |
| (...skipping 805 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1523 __ ret(); | 1458 __ ret(); |
| 1524 return true; | 1459 return true; |
| 1525 } | 1460 } |
| 1526 | 1461 |
| 1527 | 1462 |
| 1528 #undef __ | 1463 #undef __ |
| 1529 | 1464 |
| 1530 } // namespace dart | 1465 } // namespace dart |
| 1531 | 1466 |
| 1532 #endif // defined TARGET_ARCH_X64 | 1467 #endif // defined TARGET_ARCH_X64 |
| OLD | NEW |