feature: add support for wait event preamble in append command list

- add mechanism in queue to trigger start command from queue to regular
- add detection in immediate command list need to dispatch extra start in queue
- fix secondary linear stream in immediate case as it should not use container
- modify tests for primary batch buffer dispatch as default mode
- remove invalid or obsolete tests

Related-To: NEO-10356

Signed-off-by: Zbigniew Zdanowicz <zbigniew.zdanowicz@intel.com>

Author: zzdanowicz

level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl

@@ -438,7 +438,9 @@ inline ze_result_t CommandListCoreFamilyImmediate<gfxCoreFamily>::executeCommand
     size_t commandStreamStart = this->cmdListCurrentStartOffset;
     if (appendOperation == NEO::AppendOperations::cmdList && this->dispatchCmdListBatchBufferAsPrimary) {
         auto cmdListStartCmdBufferStream = reinterpret_cast<CommandQueueImp *>(cmdQ)->getStartingCmdBuffer();
-        // check if queue starting stream is the same as immediate, if not - regular cmdlist is the starting command buffer
+        // check if queue starting stream is the same as immediate,
+        // if they are the same - immediate command list buffer has preamble in it including jump from immediate to regular cmdlist - proceed normal
+        // if not - regular cmdlist is the starting command buffer - no queue preamble or waiting commands
         if (cmdListStartCmdBufferStream != commandStream) {
             commandStream = cmdListStartCmdBufferStream;
             commandStreamStart = 0u;
@@ -1720,7 +1722,16 @@ ze_result_t CommandListCoreFamilyImmediate<gfxCoreFamily>::appendCommandLists(ui
         return ret;
     }
 
-    auto mainAppendLock = static_cast<CommandQueueImp *>(this->cmdQImmediate)->getCsr()->obtainUniqueOwnership();
+    auto queueImp = static_cast<CommandQueueImp *>(this->cmdQImmediate);
+
+    auto mainAppendLock = queueImp->getCsr()->obtainUniqueOwnership();
+
+    if (this->dispatchCmdListBatchBufferAsPrimary) {
+        // check if wait event preamble or implicit synchronization is present and force bb start jump in queue, even when no preamble is required there
+        if (this->commandContainer.getCommandStream()->getUsed() != this->cmdListCurrentStartOffset) {
+            queueImp->triggerBbStartJump();
+        }
+    }
     ret = this->cmdQImmediate->executeCommandLists(numCommandLists, phCommandLists, nullptr, true, this->commandContainer.getCommandStream());
     if (ret != ZE_RESULT_SUCCESS) {
         return ret;

level_zero/core/source/cmdqueue/cmdqueue_hw.inl

@@ -253,7 +253,7 @@ size_t CommandQueueHw<gfxCoreFamily>::estimateStreamSizeForExecuteCommandListsRe
         linearStreamSizeEstimate += NEO::EncodeBatchBufferStartOrEnd<GfxFamily>::getBatchBufferEndSize();
     }
 
-    linearStreamSizeEstimate += this->estimateCommandListPrimaryStart(ctx.globalInit);
+    linearStreamSizeEstimate += this->estimateCommandListPrimaryStart(ctx.globalInit || this->forceBbStartJump);
     for (uint32_t i = 0; i < numCommandLists; i++) {
         auto cmdList = CommandList::fromHandle(commandListHandles[i]);
         linearStreamSizeEstimate += estimateCommandListSecondaryStart(cmdList);
@@ -486,7 +486,7 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandListsCopyOnly(
         ctx.spaceForResidency += estimateCommandListResidencySize(commandList);
     }
 
-    linearStreamSizeEstimate += this->estimateCommandListPrimaryStart(ctx.globalInit);
+    linearStreamSizeEstimate += this->estimateCommandListPrimaryStart(ctx.globalInit || this->forceBbStartJump);
     if (fenceRequired) {
         linearStreamSizeEstimate += NEO::MemorySynchronizationCommands<GfxFamily>::getSizeForSingleAdditionalSynchronization(device->getNEODevice()->getRootDeviceEnvironment());
     }
@@ -1020,7 +1020,7 @@ size_t CommandQueueHw<gfxCoreFamily>::estimateLinearStreamSizeComplementary(
     }
 
     bool firstCmdlistDynamicPreamble = (this->stateChanges.size() > 0 && this->stateChanges[0].cmdListIndex == 0);
-    bool estimateBbStartForGlobalInitOnly = !firstCmdlistDynamicPreamble && ctx.globalInit;
+    bool estimateBbStartForGlobalInitOnly = !firstCmdlistDynamicPreamble && (ctx.globalInit || this->forceBbStartJump);
     linearStreamSizeEstimate += this->estimateCommandListPrimaryStart(estimateBbStartForGlobalInitOnly);
 
     return linearStreamSizeEstimate;
@@ -1217,7 +1217,7 @@ void CommandQueueHw<gfxCoreFamily>::programOneCmdListBatchBufferStartPrimaryBatc
     auto bbStartPatchLocation = reinterpret_cast<MI_BATCH_BUFFER_START *>(ctx.currentPatchForChainedBbStart);
 
     bool dynamicPreamble = ctx.childGpuAddressPositionBeforeDynamicPreamble != commandStream.getCurrentGpuAddressPosition();
-    if (ctx.globalInit || dynamicPreamble) {
+    if (ctx.globalInit || dynamicPreamble || this->forceBbStartJump) {
         if (ctx.currentPatchForChainedBbStart) {
             // dynamic preamble, 2nd or later command list
             // jump from previous command list to the position before dynamic preamble
@@ -1230,6 +1230,7 @@ void CommandQueueHw<gfxCoreFamily>::programOneCmdListBatchBufferStartPrimaryBatc
         NEO::EncodeBatchBufferStartOrEnd<GfxFamily>::programBatchBufferStart(&commandStream, cmdListFirstCmdBuffer->getGpuAddress(), false, false, false);
 
         ctx.globalInit = false;
+        this->forceBbStartJump = false;
     } else {
         if (ctx.currentPatchForChainedBbStart == nullptr) {
             // nothing to dispatch from queue, first command list will be used as submitting batch buffer to KMD or ULLS

level_zero/core/source/cmdqueue/cmdqueue_imp.h

@@ -114,6 +114,9 @@ struct CommandQueueImp : public CommandQueue {
     NEO::LinearStream *getStartingCmdBuffer() const {
         return startingCmdBuffer;
     }
+    void triggerBbStartJump() {
+        forceBbStartJump = true;
+    }
 
   protected:
     MOCKABLE_VIRTUAL NEO::SubmissionStatus submitBatchBuffer(size_t offset, NEO::ResidencyContainer &residencyContainer, void *endingCmdPtr,
@@ -171,6 +174,7 @@ struct CommandQueueImp : public CommandQueue {
 
     std::atomic<bool> cmdListWithAssertExecuted = false;
     bool useKmdWaitFunction = false;
+    bool forceBbStartJump = false;
 };
 
 } // namespace L0

level_zero/core/test/unit_tests/mocks/mock_cmdlist.h

@@ -92,6 +92,7 @@ struct WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>
     using BaseClass::inOrderAtomicSignalingEnabled;
     using BaseClass::inOrderExecInfo;
     using BaseClass::inOrderPatchCmds;
+    using BaseClass::internalUsage;
     using BaseClass::interruptEvents;
     using BaseClass::isFlushTaskSubmissionEnabled;
     using BaseClass::isInOrderNonWalkerSignalingRequired;
@@ -217,6 +218,7 @@ struct WhiteBox<L0::CommandListCoreFamilyImmediate<gfxCoreFamily>>
     using BaseClass::inOrderAtomicSignalingEnabled;
     using BaseClass::inOrderExecInfo;
     using BaseClass::inOrderPatchCmds;
+    using BaseClass::internalUsage;
     using BaseClass::interruptEvents;
     using BaseClass::isBcsSplitNeeded;
     using BaseClass::isFlushTaskSubmissionEnabled;
@@ -264,6 +266,7 @@ struct MockCommandListImmediate : public CommandListCoreFamilyImmediate<gfxCoreF
     using BaseClass::finalStreamState;
     using BaseClass::immediateCmdListHeapSharing;
     using BaseClass::indirectAllocationsAllowed;
+    using BaseClass::internalUsage;
     using BaseClass::isFlushTaskSubmissionEnabled;
     using BaseClass::isSyncModeQueue;
     using BaseClass::isTbxMode;
@@ -708,6 +711,7 @@ class MockCommandListImmediateHw : public WhiteBox<::L0::CommandListCoreFamilyIm
     using BaseClass::dcFlushSupport;
     using BaseClass::dependenciesPresent;
     using BaseClass::dummyBlitWa;
+    using BaseClass::internalUsage;
     using BaseClass::isFlushTaskSubmissionEnabled;
     using BaseClass::isSyncModeQueue;
     using BaseClass::isTbxMode;

level_zero/core/test/unit_tests/mocks/mock_cmdqueue.h

@@ -31,6 +31,7 @@ struct WhiteBox<::L0::CommandQueue> : public ::L0::CommandQueueImp {
     using BaseClass::desc;
     using BaseClass::device;
     using BaseClass::firstCmdListStream;
+    using BaseClass::forceBbStartJump;
     using BaseClass::preemptionCmdSyncProgramming;
     using BaseClass::printfKernelContainer;
     using BaseClass::startingCmdBuffer;
@@ -77,6 +78,7 @@ struct MockCommandQueueHw : public L0::CommandQueueHw<gfxCoreFamily> {
     using BaseClass::commandStream;
     using BaseClass::estimateStreamSizeForExecuteCommandListsRegularHeapless;
     using BaseClass::executeCommandListsRegularHeapless;
+    using BaseClass::forceBbStartJump;
     using BaseClass::prepareAndSubmitBatchBuffer;
     using BaseClass::printfKernelContainer;
     using BaseClass::startingCmdBuffer;

level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_4.cpp

@@ -1584,8 +1584,8 @@ HWTEST2_F(ImmediateCommandListTest, givenImmediateCmdListWhenAppendingRegularThe
     auto startStream = static_cast<L0::CommandQueueImp *>(commandListImmediate->cmdQImmediate)->getStartingCmdBuffer();
 
     if (commandListImmediate->getCmdListBatchBufferFlag()) {
-        auto expectedStream = commandList->getCmdContainer().getCommandStream();
-        EXPECT_EQ(expectedStream, startStream);
+        auto expectedStreamAllocation = commandList->getCmdContainer().getCommandStream()->getGraphicsAllocation();
+        EXPECT_EQ(expectedStreamAllocation, startStream->getGraphicsAllocation());
     } else {
         auto expectedStream = commandListImmediate->getCmdContainer().getCommandStream();
         EXPECT_EQ(expectedStream, startStream);
@@ -1696,5 +1696,66 @@ HWTEST2_F(ImmediateCommandListTest,
     EXPECT_NE(cmdList.end(), iterator);
 }
 
+HWTEST2_F(ImmediateCommandListTest,
+          givenImmediateCmdListWithPrimaryBatchBufferWhenAppendingRegularCmdListWithWaitEventThenDispatchSemaphoreAndJumpFromImmediateToRegular, MatchAny) {
+    using MI_BATCH_BUFFER_START = typename FamilyType::MI_BATCH_BUFFER_START;
+    using MI_SEMAPHORE_WAIT = typename FamilyType::MI_SEMAPHORE_WAIT;
+
+    ze_event_pool_desc_t eventPoolDesc = {ZE_STRUCTURE_TYPE_EVENT_POOL_DESC};
+    eventPoolDesc.flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
+    eventPoolDesc.count = 2;
+
+    ze_event_desc_t eventDesc = {ZE_STRUCTURE_TYPE_EVENT_DESC};
+    eventDesc.index = 0;
+    eventDesc.wait = 0;
+    eventDesc.signal = 0;
+
+    ze_result_t returnValue;
+    std::unique_ptr<EventPool> eventPool = std::unique_ptr<EventPool>(static_cast<EventPool *>(EventPool::create(driverHandle.get(), context, 0, nullptr, &eventPoolDesc, returnValue)));
+    std::unique_ptr<L0::Event> event = std::unique_ptr<L0::Event>(getHelper<L0GfxCoreHelper>().createEvent(eventPool.get(), &eventDesc, device));
+    auto eventHandle = event->toHandle();
+
+    commandList->close();
+    auto cmdListHandle = commandList->toHandle();
+
+    auto regularCmdBufferStream = commandList->getCmdContainer().getCommandStream();
+    auto regularCmdBufferAllocation = regularCmdBufferStream->getGraphicsAllocation();
+
+    auto cmdQImmediate = static_cast<WhiteBox<::L0::CommandQueue> *>(commandListImmediate->cmdQImmediate);
+
+    commandListImmediate->dispatchCmdListBatchBufferAsPrimary = true;
+    cmdQImmediate->dispatchCmdListBatchBufferAsPrimary = true;
+
+    // first append can carry preamble
+    returnValue = commandListImmediate->appendCommandLists(1, &cmdListHandle, nullptr, 0, nullptr);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, returnValue);
+
+    auto immediateCmdBufferStream = commandListImmediate->getCmdContainer().getCommandStream();
+    auto offsetBefore = immediateCmdBufferStream->getUsed();
+
+    // no preamble but wait event as first, then bb_start jumping to regular cmdlist
+    returnValue = commandListImmediate->appendCommandLists(1, &cmdListHandle, nullptr, 1, &eventHandle);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, returnValue);
+
+    auto offsetAfter = immediateCmdBufferStream->getUsed();
+
+    GenCmdList cmdList;
+    ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer(
+        cmdList,
+        ptrOffset(immediateCmdBufferStream->getCpuBase(), offsetBefore),
+        offsetAfter - offsetBefore));
+
+    auto iteratorWait = find<MI_SEMAPHORE_WAIT *>(cmdList.begin(), cmdList.end());
+    ASSERT_NE(cmdList.end(), iteratorWait);
+
+    auto iteratorBbStart = find<MI_BATCH_BUFFER_START *>(iteratorWait, cmdList.end());
+    ASSERT_NE(cmdList.end(), iteratorBbStart);
+
+    auto bbStart = genCmdCast<MI_BATCH_BUFFER_START *>(*iteratorBbStart);
+
+    EXPECT_EQ(regularCmdBufferAllocation->getGpuAddress(), bbStart->getBatchBufferStartAddress());
+    EXPECT_EQ(MI_BATCH_BUFFER_START::SECOND_LEVEL_BATCH_BUFFER::SECOND_LEVEL_BATCH_BUFFER_FIRST_LEVEL_BATCH, bbStart->getSecondLevelBatchBuffer());
+}
+
 } // namespace ult
 } // namespace L0

level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_3.cpp

@@ -1201,52 +1201,5 @@ HWTEST2_F(MultiTileImmediateCommandListAppendLaunchKernelXeHpCoreTest, givenImpl
     EXPECT_EQ(cmdList.end(), itorSemaphoreWait);
 }
 
-HWTEST2_F(MultiTileImmediateCommandListAppendLaunchKernelXeHpCoreTest, givenImplicitScalingWhenUsingImmediateCommandListWithoutFlushTaskThenUseSecondaryBuffer, IsAtLeastXeHpCore) {
-    using WalkerVariant = typename FamilyType::WalkerVariant;
-
-    using MI_BATCH_BUFFER_START = typename FamilyType::MI_BATCH_BUFFER_START;
-
-    debugManager.flags.UsePipeControlAfterPartitionedWalker.set(1);
-
-    ze_group_count_t groupCount{128, 1, 1};
-
-    ze_command_queue_desc_t queueDesc = {};
-    auto queue = std::make_unique<Mock<CommandQueue>>(device, device->getNEODevice()->getDefaultEngine().commandStreamReceiver, &queueDesc);
-
-    auto immediateCmdList = std::make_unique<WhiteBox<::L0::CommandListCoreFamily<gfxCoreFamily>>>();
-    immediateCmdList->cmdListType = ::L0::CommandList::CommandListType::typeImmediate;
-    immediateCmdList->isFlushTaskSubmissionEnabled = false;
-    immediateCmdList->cmdQImmediate = queue.get();
-    auto result = immediateCmdList->initialize(device, NEO::EngineGroupType::compute, 0u);
-    ASSERT_EQ(ZE_RESULT_SUCCESS, result);
-
-    auto cmdStream = immediateCmdList->getCmdContainer().getCommandStream();
-
-    auto sizeBefore = cmdStream->getUsed();
-    CmdListKernelLaunchParams launchParams = {};
-    result = immediateCmdList->appendLaunchKernelWithParams(kernel.get(), groupCount, nullptr, launchParams);
-    ASSERT_EQ(ZE_RESULT_SUCCESS, result);
-    auto sizeAfter = cmdStream->getUsed();
-
-    GenCmdList cmdList;
-    ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer(
-        cmdList,
-        ptrOffset(cmdStream->getCpuBase(), sizeBefore),
-        sizeAfter - sizeBefore));
-
-    auto itorWalker = NEO::UnitTestHelper<FamilyType>::findWalkerTypeCmd(cmdList.begin(), cmdList.end());
-    ASSERT_NE(cmdList.end(), itorWalker);
-
-    WalkerVariant walkerCmd = NEO::UnitTestHelper<FamilyType>::getWalkerVariant(*itorWalker);
-    std::visit([](auto &&walker) {
-        EXPECT_TRUE(walker->getWorkloadPartitionEnable());
-    },
-               walkerCmd);
-
-    auto itorBbStart = find<MI_BATCH_BUFFER_START *>(cmdList.begin(), cmdList.end());
-    ASSERT_NE(cmdList.end(), itorBbStart);
-    auto cmdBbStart = genCmdCast<MI_BATCH_BUFFER_START *>(*itorBbStart);
-    EXPECT_EQ(MI_BATCH_BUFFER_START::SECOND_LEVEL_BATCH_BUFFER::SECOND_LEVEL_BATCH_BUFFER_SECOND_LEVEL_BATCH, cmdBbStart->getSecondLevelBatchBuffer());
-}
 } // namespace ult
 } // namespace L0

level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_wait_on_events.cpp

@@ -214,16 +214,23 @@ HWTEST2_F(CommandListAppendWaitOnEvent, givenImmediateCmdListAndAppendingRegular
     std::unique_ptr<L0::CommandList> commandListRegular(CommandList::create(productFamily, device, NEO::EngineGroupType::compute, 0u, returnValue, false));
     commandListRegular->close();
     auto commandListHandle = commandListRegular->toHandle();
+
+    // 1st append can carry preamble
     auto result = immCommandList->appendCommandLists(1u, &commandListHandle, nullptr, 1u, &hEventHandle);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, result);
+
+    // 2nd append should carry only wait events and bb_start to regular command list
+    auto usedSpaceBefore = immCommandList->getCmdContainer().getCommandStream()->getUsed();
 
+    result = immCommandList->appendCommandLists(1u, &commandListHandle, nullptr, 1u, &hEventHandle);
     EXPECT_EQ(ZE_RESULT_SUCCESS, result);
 
     auto usedSpaceAfter = immCommandList->getCmdContainer().getCommandStream()->getUsed();
 
     GenCmdList cmdList;
     ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer(cmdList,
-                                                      immCommandList->getCmdContainer().getCommandStream()->getCpuBase(),
-                                                      usedSpaceAfter));
+                                                      ptrOffset(immCommandList->getCmdContainer().getCommandStream()->getCpuBase(), usedSpaceBefore),
+                                                      usedSpaceAfter - usedSpaceBefore));
 
     auto itor = find<MI_SEMAPHORE_WAIT *>(cmdList.begin(), cmdList.end());
     ASSERT_NE(cmdList.end(), itor);

level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue_1.cpp

@@ -540,48 +540,6 @@ HWTEST_F(CommandQueueCreate, givenUpdateTaskCountFromWaitAndRegularCmdListWhenDi
     commandQueue->destroy();
 }
 
-HWTEST_F(CommandQueueCreate, givenUpdateTaskCountFromWaitAndImmediateCmdListWhenDispatchTaskCountWriteThenNoPipeControlFlushed) {
-    using PIPE_CONTROL = typename FamilyType::PIPE_CONTROL;
-    using POST_SYNC_OPERATION = typename FamilyType::PIPE_CONTROL::POST_SYNC_OPERATION;
-
-    DebugManagerStateRestore restorer;
-    debugManager.flags.UpdateTaskCountFromWait.set(3);
-
-    const ze_command_queue_desc_t desc = {};
-    ze_result_t returnValue;
-    auto commandQueue = whiteboxCast(CommandQueue::create(productFamily,
-                                                          device,
-                                                          neoDevice->getDefaultEngine().commandStreamReceiver,
-                                                          &desc,
-                                                          false,
-                                                          false,
-                                                          true,
-                                                          returnValue));
-
-    auto commandList = CommandList::createImmediate(productFamily, device, &desc, false, NEO::EngineGroupType::renderCompute, returnValue);
-    ASSERT_NE(nullptr, commandList);
-
-    ze_command_list_handle_t cmdListHandle = commandList->toHandle();
-    commandQueue->executeCommandLists(1, &cmdListHandle, nullptr, false, nullptr);
-
-    GenCmdList cmdList;
-    ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer(
-        cmdList, ptrOffset(commandQueue->commandStream.getCpuBase(), 0), commandQueue->commandStream.getUsed()));
-
-    auto pipeControls = findAll<PIPE_CONTROL *>(cmdList.begin(), cmdList.end());
-    bool pipeControlsPostSync = false;
-    for (size_t i = 0; i < pipeControls.size(); i++) {
-        auto pipeControl = reinterpret_cast<PIPE_CONTROL *>(*pipeControls[i]);
-        if (pipeControl->getPostSyncOperation() == POST_SYNC_OPERATION::POST_SYNC_OPERATION_WRITE_IMMEDIATE_DATA) {
-            pipeControlsPostSync = true;
-        }
-    }
-    EXPECT_FALSE(pipeControlsPostSync);
-
-    commandList->destroy();
-    commandQueue->destroy();
-}
-
 HWTEST_F(CommandQueueCreate, givenContainerWithAllocationsWhenResidencyContainerIsEmptyThenMakeResidentWasNotCalled) {
     auto csr = std::make_unique<MockCommandStreamReceiver>(*neoDevice->getExecutionEnvironment(), 0, neoDevice->getDeviceBitfield());
     csr->setupContext(*neoDevice->getDefaultEngine().osContext);

shared/source/command_container/cmdcontainer.cpp

@@ -112,7 +112,7 @@ CommandContainer::ErrorCode CommandContainer::initialize(Device *device, Allocat
             return ErrorCode::outOfDeviceMemory;
         }
         secondaryCommandStreamForImmediateCmdList = std::make_unique<LinearStream>(cmdBufferAllocationHost->getUnderlyingBuffer(),
-                                                                                   usableSize, this, this->selectedBbCmdSize);
+                                                                                   usableSize, cmdcontainer, this->selectedBbCmdSize);
         secondaryCommandStreamForImmediateCmdList->replaceGraphicsAllocation(cmdBufferAllocationHost);
         cmdBufferAllocations.push_back(cmdBufferAllocationHost);
         addToResidencyContainer(cmdBufferAllocationHost);