Bug - GraphQLError: Execution timeout, too many concurrent call on the same entities / File not found or restricted

[baptiste-fourmont]

Description

Since the 6.4.3 version i got some problems of concurrent call on the same entities, or missing file not found or restricted error.

Environment

1. OS (where OpenCTI server runs): { e.g. Mac OS 10, Windows 10, Ubuntu 16.4, etc. } RHEL8 (Cluster)
2. OpenCTI version: { e.g. OpenCTI 1.0.2 } 6.4.4
3. OpenCTI client: { e.g. frontend or python }
4. Other environment details:

Reproducible Steps

Steps to create the smallest reproducible scenario:

1. { e.g. Run ... }
2. { e.g. Click ... }
3. { e.g. Error ... }

Expected Output

Actual Output

"category":"APP","errors":[{"attributes":{"filename":"import/Artifact/e9359d18-6203-40a1-83b8-2847099b9b88/file","genre":"BUSINESS","http_status":400},"message":"File not found or restricted","name":"FUNCTIONAL_ERROR","stack":"GraphQLError: File not found or restricted\n    at error (/opt/opencti/build/src/config/errors.js:7:10)\n    at FunctionalError (/opt/opencti/build/src/config/errors.js:94:50)\n    at loadFile (/opt/opencti/build/src/database/file-storage.js:276:15)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at deleteFile (/opt/opencti/build/src/database/file-storage.js:118:14)\n    at deleteFiles (/opt/opencti/build/src/database/file-storage.js:145:5)\n    at processDeleteOperation (/opt/opencti/build/src/modules/deleteOperation/deleteOperation-domain.ts:256:7)\n    at confirmDelete (/opt/opencti/build/src/modules/deleteOperation/deleteOperation-domain.ts:271:12)\n    at Object.garbageCollectionHandler [as handler] (/opt/opencti/build/src/manager/garbageCollectionManager.ts:25:7)\n    at cronHandler (/opt/opencti/build/src/manager/managerModule.ts:75:11)\n    at /opt/opencti/build/src/manager/managerModule.ts:132:11\n    at zit.#runHandlerAndScheduleTimeout (/opt/opencti/build/node_modules/set-interval-async/dist/set-interval-async-timer.cjs:36:13)\n    at Timeout._onTimeout (/opt/opencti/build/node_modules/set-interval-async/dist/set-interval-async-timer.cjs:29:13)"}],"id":"a8f42f5a-9fce-4649-a07b-9b2feb03dcfe","level":"warn","manager":"GARBAGE_COLLECTION_MANAGER","message":"File not found or restricted","source":"backend","timestamp":"2024-12-18T16:16:50.602Z","version":"6.4.4"}

"category":"APP","errors":[{"attributes":{"genre":"BUSINESS","http_status":500,"participantIds":["relationship------ etc etc "]},"message":"Execution timeout, too many concurrent call on the same entities","name":"LOCK_ERROR","stack":"GraphQLError: Execution timeout, too many concurrent call on the same entities\n    at error (/opt/opencti/build/src/config/errors.js:7:10)\n    at LockTimeoutError (/opt/opencti/build/src/config/errors.js:109:51)\n    at createRelationRaw (/opt/opencti/build/src/database/middleware.js:2882:13)\n    at processTicksAndRe

Additional information

Screenshots (optional)

[baptiste-fourmont]

Since the migration i got error like: 'Update indexing fail'

[nino-filigran]

@baptiste-fourmont would you be able to give us a reproduction scenario so that we can try to reproduce it? When do you get this error regarding your artifact? Is that something you deleted ? Also we just released a new version, so you might want to upgrade to the latest and see if the problem persists.

[baptiste-fourmont]

Hello I'm going to upgrade to 6.4.5. For the artifact, it has probably been removed by the retention policy.
I'll give you feedback as soon as I've done the upgrade

[baptiste-fourmont]

Good morning,
I migrated to 6.4.5 but it seems that the error is not resolved.
It seems that the errors below block "updating the elasticsearch index".
This type of error happened with there 6.4.3.

It seems that the error is only caused on the artifcat object type

{"category":"APP","errors":[{"attributes":{"filename":"import/Artifact/5e36c103-7c6b-4cb9-9630-029e14582505/5-2839-2024_01.09.2024.lnk","genre":"BUSINESS","http_status":400},"message":"File not found or restricted","name":"FUNCTIONAL_ERROR","stack":"GraphQLError: File not found or restricted\n    at error (/opt/opencti/build/src/config/errors.js:7:10)\n    at FunctionalError (/opt/opencti/build/src/config/errors.js:94:50)\n    at loadFile (/opt/opencti/build/src/database/file-storage.js:301:15)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at deleteFile (/opt/opencti/build/src/database/file-storage.js:126:14)\n    at deleteFiles (/opt/opencti/build/src/database/file-storage.js:153:5)\n    at processDeleteOperation (/opt/opencti/build/src/modules/deleteOperation/deleteOperation-domain.ts:256:7)\n    at confirmDelete (/opt/opencti/build/src/modules/deleteOperation/deleteOperation-domain.ts:271:12)\n    at Object.garbageCollectionHandler [as handler] (/opt/opencti/build/src/manager/garbageCollectionManager.ts:25:7)\n    at cronHandler (/opt/opencti/build/src/manager/managerModule.ts:75:11)\n    at /opt/opencti/build/src/manager/managerModule.ts:132:11\n    at Dnt.#runHandlerAndScheduleTimeout (/opt/opencti/build/node_modules/set-interval-async/dist/set-interval-async-timer.cjs:36:13)\n    at Timeout._onTimeout (/opt/opencti/build/node_modules/set-interval-async/dist/set-interval-async-timer.cjs:29:13)"}],"id":"4eaeba1b-7549-4e9d-8c39-52ea3f2787fe","level":"warn","manager":"GARBAGE_COLLECTION_MANAGER","message":"File not found or restricted","source":"backend","timestamp":"2024-12-19T10:02:44.695Z","version":"6.4.5"}
{"category":"APP","errors":[{"attributes":{"filename":"import/Artifact/ad1a4dce-bb54-4578-9986-2443ccd7129a/RFQ-MR-24-09101 SPS.js","genre":"BUSINESS","http_status":400},"message":"File not found or restricted","name":"FUNCTIONAL_ERROR","stack":"GraphQLError: File not found or restricted\n    at error (/opt/opencti/build/src/config/errors.js:7:10)\n    at FunctionalError (/opt/opencti/build/src/config/errors.js:94:50)\n    at loadFile (/opt/opencti/build/src/database/file-storage.js:301:15)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at deleteFile (/opt/opencti/build/src/database/file-storage.js:126:14)\n    at deleteFiles (/opt/opencti/build/src/database/file-storage.js:153:5)\n    at processDeleteOperation (/opt/opencti/build/src/modules/deleteOperation/deleteOperation-domain.ts:256:7)\n    at confirmDelete (/opt/opencti/build/src/modules/deleteOperation/deleteOperation-domain.ts:271:12)\n    at Object.garbageCollectionHandler [as handler] (/opt/opencti/build/src/manager/garbageCollectionManager.ts:25:7)\n    at cronHandler (/opt/opencti/build/src/manager/managerModule.ts:75:11)\n    at /opt/opencti/build/src/manager/managerModule.ts:132:11\n    at Dnt.#runHandlerAndScheduleTimeout (/opt/opencti/build/node_modules/set-interval-async/dist/set-interval-async-timer.cjs:36:13)\n    at Timeout._onTimeout (/opt/opencti/build/node_modules/set-interval-async/dist/set-interval-async-timer.cjs:29:13)"}],"id":"bc64abd3-c9b8-4688-85a3-9a9a2b67244f","level":"warn","manager":"GARBAGE_COLLECTION_MANAGER","message":"File not found or restricted","source":"backend","timestamp":"2024-12-19T10:02:44.714Z","version":"6.4.5"}
{"category":"APP","errors":[{"attributes":{"filename":"import/Artifact/07c13fb9-6ed7-4a17-979d-f73658bc497a/qsKo","genre":"BUSINESS","http_status":400},"message":"File not found or restricted","name":"FUNCTIONAL_ERROR","stack":"GraphQLError: File not found or restricted\n    at error (/opt/opencti/build/src/config/errors.js:7:10)\n    at FunctionalError (/opt/opencti/build/src/config/errors.js:94:50)\n    at loadFile (/opt/opencti/build/src/database/file-storage.js:301:15)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at deleteFile (/opt/opencti/build/src/database/file-storage.js:126:14)\n    at deleteFiles (/opt/opencti/build/src/database/file-storage.js:153:5)\n    at processDeleteOperation (/opt/opencti/build/src/modules/deleteOperation/deleteOperation-domain.ts:256:7)\n    at confirmDelete (/opt/opencti/build/src/modules/deleteOperation/deleteOperation-domain.ts:271:12)\n    at Object.garbageCollectionHandler [as handler] (/opt/opencti/build/src/manager/garbageCollectionManager.ts:25:7)\n    at cronHandler (/opt/opencti/build/src/manager/managerModule.ts:75:11)\n    at /opt/opencti/build/src/manager/managerModule.ts:132:11\n    at Dnt.#runHandlerAndScheduleTimeout (/opt/opencti/build/node_modules/set-interval-async/dist/set-interval-async-timer.cjs:36:13)\n    at Timeout._onTimeout (/opt/opencti/build/node_modules/set-interval-async/dist/set-interval-async-timer.cjs:29:13)"}],"id":"a8376033-8947-421c-9222-0d5702539b78","level":"warn","manager":"GARBAGE_COLLECTION_MANAGER","message":"File not found or restricted","source":"backend","timestamp":"2024-12-19T10:02:44.727Z","version":"6.4.5"}
{"category":"APP","errors":[{"attributes":{"filename":"import/Artifact/2a780408-69a9-4c37-8787-67b202582d0a/CSDE-DS-2024-027.exe","genre":"BUSINESS","http_status":400},"message":"File not found or restricted","name":"FUNCTIONAL_ERROR","stack":"GraphQLError: File not found or restricted\n    at error (/opt/opencti/build/src/config/errors.js:7:10)\n    at FunctionalError (/opt/opencti/build/src/config/errors.js:94:50)\n    at loadFile (/opt/opencti/build/src/database/file-storage.js:301:15)\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at deleteFile (/opt/opencti/build/src/database/file-storage.js:126:14)\n    at deleteFiles (/opt/opencti/build/src/database/file-storage.js:153:5)\n    at processDeleteOperation (/opt/opencti/build/src/modules/deleteOperation/deleteOperation-domain.ts:256:7)\n    at confirmDelete (/opt/opencti/build/src/modules/deleteOperation/deleteOperation-domain.ts:271:12)\n    at Object.garbageCollectionHandler [as handler] (/opt/opencti/build/src/manager/garbageCollectionManager.ts:25:7)\n    at cronHandler (/opt/opencti/build/src/manager/managerModule.ts:75:11)\n    at /opt/opencti/build/src/manager/managerModule.ts:132:11\n    at Dnt.#runHandlerAndScheduleTimeout (/opt/opencti/build/node_modules/set-interval-async/dist/set-interval-async-timer.cjs:36:13)\n    at Timeout._onTimeout (/opt/opencti/build/node_modules/set-interval-async/dist/set-interval-async-timer.cjs:29:13)"}],"id":"d8d3d5b0-3b4a-44c9-aa57-22616ad46382","level":"warn","manager":"GARBAGE_COLLECTION_MANAGER","message":"File not found or restricted","source":"backend","timestamp":"2024-12-19T10:02:44.739Z","version":"6.4.5"}

[baptiste-fourmont]

It's seems to be linked to: #9424

[labo-flg]

Hi !
The artifact files mentioned in the errors, do you confirm these files are not present anymore on your storage ?
We shall fix the garbage collector manager so it won't fail in that case as the data is already removed.

Now, I'm not sure how it could lead to failed initialization of the platform. The manager just runs every 10 minutes and fails with error logs and that's it.

[baptiste-fourmont]

Hello,

I can confirm that they are no longer there. I've checked minio and they're no longer there. I saw that there was this error on startup. And each time the retention manager or garabage manager make actions it and it rertuns warning of filename: import/Artifact.

{"category":"APP","count":1111,"errorCount":1111,"level":"error","message":"[OPENCTI-MODULE] Garbage collection manager had errors during cleanup. Please have a look at previous warnings.","source":"backend","timestamp":"2024-12-27T11:00:11.779Z","version":"6.4.5"}
{"category":"APP","errors":[{"attributes":{"errors":[{"index":"opencti_stix_meta_objects-000001","index_uuid":"t4PKJwF-RoaRlKfwOmgaSw","reason":"[3633fb06-773f-4cfb-a117-ec16574c2e67]: version conflict, required seqNo [2215798], primary term [63]. current document has seqNo [2215801] and primary term [63]","shard":"0","type":"version_conflict_engine_exception"}],"genre":"TECHNICAL","http_status":500},"message":"Bulk indexing fail","name":"DATABASE_ERROR","stack":"GraphQLError: Bulk indexing fail\n    at error (/opt/opencti/build/src/config/errors.js:7:10)\n    at DatabaseError (/opt/opencti/build/src/config/errors.js:57:48)\n    at /opt/opencti/build/src/database/engine.js:3421:13\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at elRemoveRelationConnection (/opt/opencti/build/src/database/engine.js:3578:9)\n    at elDeleteElements (/opt/opencti/build/src/database/engine.js:3769:3)\n    at internalDeleteElementById (/opt/opencti/build/src/database/middleware.js:3253:7)\n    at deleteElementById (/opt/opencti/build/src/database/middleware.js:3275:32)\n    at deleteElement (/opt/opencti/build/src/manager/retentionManager.ts:37:5)\n    at deleteFn (/opt/opencti/build/src/manager/retentionManager.ts:83:11)"}],"id":"1f0487ff-732f-4f24-bc13-d1edb35a0458","level":"error","manager":"RETENTION_MANAGER","message":"Bulk indexing fail","source":"backend","timestamp":"2024-12-27T11:00:11.982Z","version":"6.4.5"}
{"category":"APP","errors":[{"attributes":{"errors":[{"index":"opencti_stix_meta_objects-000001","index_uuid":"t4PKJwF-RoaRlKfwOmgaSw","reason":"[3633fb06-773f-4cfb-a117-ec16574c2e67]: version conflict, required seqNo [2215803], primary term [63]. current document has seqNo [2215804] and primary term [63]","shard":"0","type":"version_conflict_engine_exception"}],"genre":"TECHNICAL","http_status":500},"message":"Bulk indexing fail","name":"DATABASE_ERROR","stack":"GraphQLError: Bulk indexing fail\n    at error (/opt/opencti/build/src/config/errors.js:7:10)\n    at DatabaseError (/opt/opencti/build/src/config/errors.js:57:48)\n    at /opt/opencti/build/src/database/engine.js:3421:13\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at elRemoveRelationConnection (/opt/opencti/build/src/database/engine.js:3578:9)\n    at elDeleteElements (/opt/opencti/build/src/database/engine.js:3769:3)\n    at internalDeleteElementById (/opt/opencti/build/src/database/middleware.js:3253:7)\n    at deleteElementById (/opt/opencti/build/src/database/middleware.js:3275:32)\n    at deleteElement (/opt/opencti/build/src/manager/retentionManager.ts:37:5)\n    at deleteFn (/opt/opencti/build/src/manager/retentionManager.ts:83:11)"}],"id":"13d87d3b-03e1-4f43-8a93-a8dbc762d888","level":"error","manager":"RETENTION_MANAGER","message":"Bulk indexing fail","source":"backend","timestamp":"2024-12-27T11:00:13.538Z","version":"6.4.5"}
{"category":"APP","errors":[{"attributes":{"errors":[{"index":"opencti_stix_meta_objects-000001","index_uuid":"t4PKJwF-RoaRlKfwOmgaSw","reason":"[3633fb06-773f-4cfb-a117-ec16574c2e67]: version conflict, required seqNo [2215840], primary term [63]. current document has seqNo [2215842] and primary term [63]","shard":"0","type":"version_conflict_engine_exception"}],"genre":"TECHNICAL","http_status":500},"message":"Bulk indexing fail","name":"DATABASE_ERROR","stack":"GraphQLError: Bulk indexing fail\n    at error (/opt/opencti/build/src/config/errors.js:7:10)\n    at DatabaseError (/opt/opencti/build/src/config/errors.js:57:48)\n    at /opt/opencti/build/src/database/engine.js:3421:13\n    at processTicksAndRejections (node:internal/process/task_queues:95:5)\n    at elRemoveRelationConnection (/opt/opencti/build/src/database/engine.js:3578:9)\n    at elDeleteElements (/opt/opencti/build/src/database/engine.js:3769:3)\n    at internalDeleteElementById (/opt/opencti/build/src/database/middleware.js:3253:7)\n    at deleteElementById (/opt/opencti/build/src/database/middleware.js:3275:32)\n    at deleteElement (/opt/opencti/build/src/manager/retentionManager.ts:37:5)\n    at deleteFn (/opt/opencti/build/src/manager/retentionManager.ts:83:11)"}],"id":"dd0c104a-e25b-46bf-b583-2bbae650eaaf","level":"error","manager":"RETENTION_MANAGER","message":"Bulk indexing fail","source":"backend","timestamp":"2024-12-27T11:00:26.206Z","version":"6.4.5"}