[server.background_client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.base.independent_thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.base.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'RDMA'
services = [ 'Me
externalClientDir = ''
externalClientPath = ''
multipleClientThreadNum = 4
readonly = false
trace_file = ''
trace_format = 'json'
[server.meta]
acl_cache_time = '15s'
allow_directly_move_to_trash = false
allow_owner_change_immutable = false
allow_stat_deleted_inodes = true
authenticate = false
batch_stat_by_path_concurrent = 4
batch_stat_concurrent = 8
check_file_hole = false
dynamic_stripe = false
dynamic_stripe_growth = 2
dynamic_stripe_initial = 16
enable_new_chunk_engine = false
grv_cache = false
idempotent_record_clean = '1min'
idempotent_record_expire = '30min'
idempotent_remove = true
idempotent_rename = false
iflags_chain_allocation = false
iflags_chunk_engine = true
ignore_length_hint = false
inodeId_abort_on_duplicate = false
inodeId_check_unique = true
list_default_limit = 128
max_batch_operations = 4096
max_directory_depth = 64
max_remove_chunks_per_request = 32
max_symlink_count = 10
max_symlink_depth = 4
operation_timeout = '5s'
otrunc_replace_file = true
otrunc_replace_file
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'
[server.storage_client.net_client.io_worker.connect_concurrency_limiter]
max_concurrency = 4
[server.storage_client.net_client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 1
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.storage_client.net_client.io_worker.transport_pool]
max_connections = 1
[server.storage_client.net_client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage_client.traffic_control.read]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage_client.traffic_control.remove]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage_client.traffic_control.truncate]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
[server.storage_client.traffic_control.write]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true NODE=node1 TASK=CreateMetaServiceTask
INFO[0022] Save admin cli config to /tmp/prepare-3fs-config.F7mgR4/admin_cli.toml NODE=node1 TASK=CreateMetaServiceTask
INFO[0022] Generating fdb.cluster to /tmp/prepare-3fs-config.F7mgR4/fdb.cluster NODE=node1 TASK=CreateMetaServiceTask
INFO[0022] Copying meta_main configs from /tmp/prepare-3fs-config.F7mgR4 to node1 /mnt/nvme3n1/wayne/opt3fs/meta/config.d NODE=node1 TASK=CreateMetaServiceTask
DEBU[0022] Run command: sudo -S /bin/bash -c cp -r /tmp/prepare-3fs-config.F7mgR4 /mnt/nvme3n1/wayne/opt3fs/meta/config.d NODE="<LOCAL>"
DEBU[0022] Output of sudo -S /bin/bash -c cp -r /tmp/prepare-3fs-config.F7mgR4 /mnt/nvme3n1/wayne/opt3fs/meta/config.d: NODE="<LOCAL>"
DEBU[0022] Run command: sudo -S /bin/bash -c rm -fr /tmp/prepare-3fs-config.F7mgR4 NODE="<LOCAL>"
DEBU[0022] Output of sudo -S /bin/bash -c rm -fr /tmp/prepare-3fs-config.F7mgR4: NODE="<LOCAL>"
INFO[0022] Upload meta_main main config NODE=node1 TASK=CreateMetaServiceTask
DEBU[0022] Run command: sudo -S /bin/bash -c docker run --name 3fs-meta-cfg-pqUyDF --network host --entrypoint '' --rm --privileged --ulimit nofile=1048576:1048576 --volume /mnt/nvme3n1/wayne/opt3fs/meta/config.d:/opt/3fs/etc/ --volume /dev:/dev open3fs/3fs:20250410 /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml --config.mgmtd_client.mgmtd_server_addresses '["RDMA://10.74.40.86:8000"]' 'set-config --type META --file /opt/3fs/etc/meta_main.toml' NODE="<LOCAL>"
DEBU[0022] Output of sudo -S /bin/bash -c docker run --name 3fs-meta-cfg-pqUyDF --network host --entrypoint '' --rm --privileged --ulimit nofile=1048576:1048576 --volume /mnt/nvme3n1/wayne/opt3fs/meta/config.d:/opt/3fs/etc/ --volume /dev:/dev open3fs/3fs:20250410 /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml --config.mgmtd_client.mgmtd_server_addresses '["RDMA://10.74.40.86:8000"]' 'set-config --type META --file /opt/3fs/etc/meta_main.toml': Succeed
ConfigVersion 1 NODE="<LOCAL>"
INFO[0022] Service meta_main main config uploaded NODE=node1 TASK=CreateMetaServiceTask
INFO[0022] Starting meta_main container 3fs-meta NODE=node1 TASK=CreateMetaServiceTask
DEBU[0022] Run command: sudo -S /bin/bash -c docker run --name 3fs-meta --detach --network host --privileged --ulimit nofile=1048576:1048576 --volume /dev:/dev --volume /mnt/nvme3n1/wayne/opt3fs/meta/config.d:/opt/3fs/etc/ --volume /mnt/nvme3n1/wayne/opt3fs/meta/log:/var/log/3fs --restart unless-stopped open3fs/3fs:20250410 /opt/3fs/bin/meta_main --launcher_cfg /opt/3fs/etc/meta_main_launcher.toml --app_cfg /opt/3fs/etc/meta_main_app.toml NODE="<LOCAL>"
DEBU[0022] Output of sudo -S /bin/bash -c docker run --name 3fs-meta --detach --network host --privileged --ulimit nofile=1048576:1048576 --volume /dev:/dev --volume /mnt/nvme3n1/wayne/opt3fs/meta/config.d:/opt/3fs/etc/ --volume /mnt/nvme3n1/wayne/opt3fs/meta/log:/var/log/3fs --restart unless-stopped open3fs/3fs:20250410 /opt/3fs/bin/meta_main --launcher_cfg /opt/3fs/etc/meta_main_launcher.toml --app_cfg /opt/3fs/etc/meta_main_app.toml: 9ac10f53d430ace271ce658087e4b48a9ad7303679f477afd4d61b5582dbeb93 NODE="<LOCAL>"
INFO[0022] Waiting for service meta_main active... NODE=node1 TASK=CreateMetaServiceTask
DEBU[0022] Run command: sudo -S /bin/bash -c docker exec 3fs-meta /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-nodes" NODE="<LOCAL>"
DEBU[0022] Output of sudo -S /bin/bash -c docker exec 3fs-meta /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-nodes": Id Type Status Hostname Pid Tags LastHeartbeatTime ConfigVersion ReleaseVersion
1 MGMTD PRIMARY_MGMTD salab-hpedl380g11-02 1 [] N/A 1(UPTODATE) 250228-dev-1-999999-ee9a5cee NODE="<LOCAL>"
DEBU[0024] Run command: sudo -S /bin/bash -c docker exec 3fs-meta /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-nodes" NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c docker exec 3fs-meta /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-nodes": Id Type Status Hostname Pid Tags LastHeartbeatTime ConfigVersion ReleaseVersion
1 MGMTD PRIMARY_MGMTD salab-hpedl380g11-02 1 [] N/A 1(UPTODATE) 250228-dev-1-999999-ee9a5cee
100 META HEARTBEAT_CONNECTED salab-hpedl380g11-02 1 [] 2026-06-07 13:28:42 0 250228-dev-1-999999-ee9a5cee NODE="<LOCAL>"
INFO[0025] Service meta_main is not active NODE=node1 TASK=CreateMetaServiceTask
INFO[0025] Started meta_main container 3fs-meta successfully NODE=node1 TASK=CreateMetaServiceTask
INFO[0025] Running task CreateStorageServiceTask
DEBU[0025] Node ID map: map[node1:10001] NODE=node1 TASK=CreateStorageServiceTask
INFO[0025] Start to run script disk_tool.sh on node NODE=node1 TASK=CreateStorageServiceTask
DEBU[0025] Run command: sudo -S /bin/bash -c mktemp -d -p /tmp -t remote-run-script.XXXXXX NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c mktemp -d -p /tmp -t remote-run-script.XXXXXX: /tmp/remote-run-script.GpIu6I NODE="<LOCAL>"
DEBU[0025] Run command: sudo -S /bin/bash -c chmod 0777 /tmp/remote-run-script.GpIu6I NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c chmod 0777 /tmp/remote-run-script.GpIu6I: NODE="<LOCAL>"
DEBU[0025] Run command: sudo -S /bin/bash -c mkdir -m 0777 -p /mnt/nvme3n1/wayne/opt3fs/storage NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c mkdir -m 0777 -p /mnt/nvme3n1/wayne/opt3fs/storage: NODE="<LOCAL>"
DEBU[0025] Run command: sudo -S /bin/bash -c mktemp -p /tmp NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c mktemp -p /tmp: /tmp/tmp.ilrcY6KA6z NODE="<LOCAL>"
DEBU[0025] Run command: sudo -S /bin/bash -c chmod 0777 /tmp/tmp.ilrcY6KA6z NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c chmod 0777 /tmp/tmp.ilrcY6KA6z: NODE="<LOCAL>"
INFO[0025] Scp /tmp/remote-run-script.GpIu6I/tmp_script.sh to /tmp/tmp.ilrcY6KA6z NODE=node1 TASK=CreateStorageServiceTask
DEBU[0025] Run command: sudo -S /bin/bash -c cp -r /tmp/remote-run-script.GpIu6I/tmp_script.sh /tmp/tmp.ilrcY6KA6z NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c cp -r /tmp/remote-run-script.GpIu6I/tmp_script.sh /tmp/tmp.ilrcY6KA6z: NODE="<LOCAL>"
INFO[0025] Run disk_tool.sh with [/mnt/nvme3n1/wayne/opt3fs/storage 1 dir prepare] NODE=node1 TASK=CreateStorageServiceTask
DEBU[0025] Run command: sudo -S /bin/bash -c bash /tmp/tmp.ilrcY6KA6z /mnt/nvme3n1/wayne/opt3fs/storage 1 dir prepare NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c bash /tmp/tmp.ilrcY6KA6z /mnt/nvme3n1/wayne/opt3fs/storage 1 dir prepare: + '[' 4 -lt 4 ']'
+ MOUNT_DIR=/mnt/nvme3n1/wayne/opt3fs/storage/3fsdata
+ DISK_NUM=1
+ DISK_TYPE=dir
+ ACTION=prepare
+ mkdir -p /mnt/nvme3n1/wayne/opt3fs/storage/3fsdata
+ [[ dir == \n\v\m\e ]]
+ [[ prepare == \p\r\e\p\a\r\e ]]
+ prepare_dir_disk
++ seq 0 0
+ for i in $(seq 0 $((DISK_NUM-1)))
+ MOUNT_POINT=/mnt/nvme3n1/wayne/opt3fs/storage/3fsdata/data0
+ mkdir -p /mnt/nvme3n1/wayne/opt3fs/storage/3fsdata/data0/3fs NODE="<LOCAL>"
DEBU[0025] Run disk_tool.sh output: + '[' 4 -lt 4 ']'
+ MOUNT_DIR=/mnt/nvme3n1/wayne/opt3fs/storage/3fsdata
+ DISK_NUM=1
+ DISK_TYPE=dir
+ ACTION=prepare
+ mkdir -p /mnt/nvme3n1/wayne/opt3fs/storage/3fsdata
+ [[ dir == \n\v\m\e ]]
+ [[ prepare == \p\r\e\p\a\r\e ]]
+ prepare_dir_disk
++ seq 0 0
+ for i in $(seq 0 $((DISK_NUM-1)))
+ MOUNT_POINT=/mnt/nvme3n1/wayne/opt3fs/storage/3fsdata/data0
+ mkdir -p /mnt/nvme3n1/wayne/opt3fs/storage/3fsdata/data0/3fs NODE=node1 TASK=CreateStorageServiceTask
INFO[0025] Run disk_tool.sh success NODE=node1 TASK=CreateStorageServiceTask
DEBU[0025] Run command: sudo -S /bin/bash -c rm -f /tmp/tmp.ilrcY6KA6z NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c rm -f /tmp/tmp.ilrcY6KA6z: NODE="<LOCAL>"
DEBU[0025] Run command: sudo -S /bin/bash -c rm -fr /tmp/remote-run-script.GpIu6I NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c rm -fr /tmp/remote-run-script.GpIu6I: NODE="<LOCAL>"
DEBU[0025] Run command: sudo -S /bin/bash -c mktemp -d -p /tmp -t prepare-3fs-config.XXXXXX NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c mktemp -d -p /tmp -t prepare-3fs-config.XXXXXX: /tmp/prepare-3fs-config.fk5gRK NODE="<LOCAL>"
DEBU[0025] Run command: sudo -S /bin/bash -c chmod 0777 /tmp/prepare-3fs-config.fk5gRK NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c chmod 0777 /tmp/prepare-3fs-config.fk5gRK: NODE="<LOCAL>"
INFO[0025] Create storage_main config dir /mnt/nvme3n1/wayne/opt3fs/storage NODE=node1 TASK=CreateStorageServiceTask
DEBU[0025] Run command: sudo -S /bin/bash -c mkdir -m 0777 -p /mnt/nvme3n1/wayne/opt3fs/storage NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c mkdir -m 0777 -p /mnt/nvme3n1/wayne/opt3fs/storage: NODE="<LOCAL>"
DEBU[0025] Template data of storage_main_app.toml.tmpl: map[NodeID:10001] NODE=node1 TASK=CreateStorageServiceTask
INFO[0025] Generating storage_main_app.toml to /tmp/prepare-3fs-config.fk5gRK/storage_main_app.toml NODE=node1 TASK=CreateStorageServiceTask
DEBU[0025] Config of storage_main_app.toml: allow_empty_node_id = true
node_id = 10001 NODE=node1 TASK=CreateStorageServiceTask
[client.rdma_control]
max_concurrent_transmission = 64
[client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[ib_devices]
allow_no_usable_devices = false
allow_unknown_zone = true
default_network_zone = 'UNKNOWN'
default_pkey_index = 0
default_roce_pkey_index = 0
default_traffic_class = 0
device_filter = []
fork_safe = true
prefer_ibdevice = true
skip_inactive_ports = true
skip_unusable_device = true
subnets = []
[mgmtd_client]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '10s'
enable_auto_extend_client_session = false
enable_auto_heartbeat = true
enable_auto_refresh = true
mgmtd_server_addresses = ["RDMA://10.74.40.86:8000"]
work_queue_size = 100 NODE=node1 TASK=CreateStorageServiceTask
[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 1
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14
[server.base.groups.io_worker.transport_pool]
max_connections = 1
[server.base.groups.listener]
domain_socket_index = 1
filter_list = []
listen_port = 8002
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false
[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
re
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'
[server.coroutines_pool_default]
coroutines_num = 64
queue_size = 1024
threads_num = 8
[server.coroutines_pool_read]
coroutines_num = 64
queue_size = 1024
threads_num = 8
[server.coroutines_pool_sync]
coroutines_num = 64
queue_size = 1024
threads_num = 8
[server.coroutines_pool_update]
coroutines_num = 64
queue_size = 1024
threads_num = 8
[server.dump_worker]
dump_interval = '1day'
dump_root_path = ''
high_cpu_usage_threshold = 100
[server.forward_client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_r
enable_work_stealing = false
queue_size = 1024
[server.targets]
allow_disk_without_uuid = true
collect_all_fds = true
create_engine_path = true
space_info_cache_timeout = '5s'
target_num_per_path = 0
target_paths = ["/mnt/3fsdata/data0/3fs"]
[server.targets.storage_target]
force_persist = true
kv_path = ''
migrate_kv_store = false
mutex_num = 257
point_query_strategy = 'NONE'
[server.targets.storage_target.file_store]
preopen_chunk_size_list = []
[server.targets.storage_target.kv_store]
create_if_missing = false
integrate_leveldb_log = false
leveldb_block_cache_size = '8GB'
leveldb_iterator_fill_cache = true
leveldb_shared_block_cache = true
leveldb_sst_file_size = '16MB'
leveldb_write_buffer_size = '16MB'
rocksdb_avoid_flush_during_recovery = false
rocksdb_avoid_flush_during_shutdown = false
rocksdb_avoid_unnecessary_blocking_io = false
rocksdb_block_cache_size = '8GB'
rocksdb_block_size = '4KB'
rocksdb_bloom_filter_bits_per_key = 10
rocksdb_compression = 'kNoCompression'
rocksdb_enable_bloom_filter = true
rocksdb_enable_pipelined_write = false
rocksdb_enable_prefix_transform = true
rocksdb_keep_log_file_num = 10
rocksdb_level0_file_num_compaction_trigger = 4
rocksdb_lowest_used_cache_tier = 'kNonVolatileBlockTier'
rocksdb_max_manifest_file_size = '64MB'
rocksdb_num_levels = 7
rocksdb_prepopulate_block_cache = 'kDisable'
rocksdb_readahead_size = '2MB'
rocksdb_shared_block_cache = true
rocksdb_stats_dump_period = '2min'
rocksdb_target_file_size_base = '64MB'
rocksdb_target_file_size_multiplier = 1
rocksdb_threads_num = 8
rocksdb_unordered_write = false
rocksdb_wal_recovery_mode = 'kTolerateCorruptedTailRecords'
rocksdb_write_buffer_size = '16MB'
sync_when_write = true
type = 'LevelDB'
[server.targets.storage_target.meta_store]
allocate_size = '256MB'
punch_hole_batch_size = 16
recycle_batch_size = 256
removed_chunk_expiration_time = '3day'
removed_chunk_force_recycled_time = '1h' NODE=node1 TASK=CreateStorageServiceTask
INFO[0025] Save admin cli config to /tmp/prepare-3fs-config.fk5gRK/admin_cli.toml NODE=node1 TASK=CreateStorageServiceTask
INFO[0025] Generating fdb.cluster to /tmp/prepare-3fs-config.fk5gRK/fdb.cluster NODE=node1 TASK=CreateStorageServiceTask
INFO[0025] Copying storage_main configs from /tmp/prepare-3fs-config.fk5gRK to node1 /mnt/nvme3n1/wayne/opt3fs/storage/config.d NODE=node1 TASK=CreateStorageServiceTask
DEBU[0025] Run command: sudo -S /bin/bash -c cp -r /tmp/prepare-3fs-config.fk5gRK /mnt/nvme3n1/wayne/opt3fs/storage/config.d NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c cp -r /tmp/prepare-3fs-config.fk5gRK /mnt/nvme3n1/wayne/opt3fs/storage/config.d: NODE="<LOCAL>"
DEBU[0025] Run command: sudo -S /bin/bash -c rm -fr /tmp/prepare-3fs-config.fk5gRK NODE="<LOCAL>"
DEBU[0025] Output of sudo -S /bin/bash -c rm -fr /tmp/prepare-3fs-config.fk5gRK: NODE="<LOCAL>"
INFO[0025] Upload storage_main main config NODE=node1 TASK=CreateStorageServiceTask
DEBU[0025] Run command: sudo -S /bin/bash -c docker run --name 3fs-storage-cfg-9SV2jE --network host --entrypoint '' --rm --privileged --ulimit nofile=1048576:1048576 --volume /mnt/nvme3n1/wayne/opt3fs/storage/config.d:/opt/3fs/etc/ --volume /dev:/dev open3fs/3fs:20250410 /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml --config.mgmtd_client.mgmtd_server_addresses '["RDMA://10.74.40.86:8000"]' 'set-config--type STORAGE --file /opt/3fs/etc/storage_main.toml' NODE="<LOCAL>"
DEBU[0026] Output of sudo -S /bin/bash -c docker run --name 3fs-storage-cfg-9SV2jE --network host --entrypoint '' --rm --privileged --ulimit nofile=1048576:1048576 --volume /mnt/nvme3n1/wayne/opt3fs/storage/config.d:/opt/3fs/etc/ --volume /dev:/dev open3fs/3fs:20250410 /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml --config.mgmtd_client.mgmtd_server_addresses '["RDMA://10.74.40.86:8000"]' 'set-config --type STORAGE --file /opt/3fs/etc/storage_main.toml': Succeed
ConfigVersion 1 NODE="<LOCAL>"
INFO[0026] Service storage_main main config uploaded NODE=node1 TASK=CreateStorageServiceTask
INFO[0026] Starting storage_main container 3fs-storage NODE=node1 TASK=CreateStorageServiceTask
DEBU[0026] Run command: sudo -S /bin/bash -c docker run --name 3fs-storage --detach --network host --privileged --ulimit nofile=1048576:1048576 --volume /dev:/dev --volume /mnt/nvme3n1/wayne/opt3fs/storage/config.d:/opt/3fs/etc/ --volume /mnt/nvme3n1/wayne/opt3fs/storage/log:/var/log/3fs --volume /mnt/nvme3n1/wayne/opt3fs/storage/3fsdata:/mnt/3fsdata --restart unless-stopped open3fs/3fs:20250410 /opt/3fs/bin/storage_main --launcher_cfg /opt/3fs/etc/storage_main_launcher.toml --app_cfg /opt/3fs/etc/storage_main_app.toml NODE="<LOCAL>"
DEBU[0026] Output of sudo -S /bin/bash -c docker run --name 3fs-storage --detach --network host --privileged --ulimit nofile=1048576:1048576 --volume /dev:/dev --volume /mnt/nvme3n1/wayne/opt3fs/storage/config.d:/opt/3fs/etc/ --volume /mnt/nvme3n1/wayne/opt3fs/storage/log:/var/log/3fs --volume /mnt/nvme3n1/wayne/opt3fs/storage/3fsdata:/mnt/3fsdata --restart unless-stopped open3fs/3fs:20250410 /opt/3fs/bin/storage_main --launcher_cfg /opt/3fs/etc/storage_main_launcher.toml --app_cfg /opt/3fs/etc/storage_main_app.toml: 0e38ac3c21b6b2a0ec48dadf750f81b851313abbbfce96df83dbed4815e65da6 NODE="<LOCAL>"
INFO[0026] Waiting for service storage_main active... NODE=node1 TASK=CreateStorageServiceTask
DEBU[0026] Run command: sudo -S /bin/bash -c docker exec 3fs-storage /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-nodes" NODE="<LOCAL>"
DEBU[0026] Output of sudo -S /bin/bash -c docker exec 3fs-storage /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-nodes": Id Type Status Hostname Pid Tags LastHeartbeatTime ConfigVersion ReleaseVersion
1 MGMTD PRIMARY_MGMTD salab-hpedl380g11-02 1 [] N/A 1(UPTODATE) 250228-dev-1-999999-ee9a5cee
100 META HEARTBEAT_CONNECTED salab-hpedl380g11-02 1 [] 2026-06-07 13:28:42 0 250228-dev-1-999999-ee9a5cee NODE="<LOCAL>"
DEBU[0028] Run command: sudo -S /bin/bash -c docker exec 3fs-storage /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-nodes" NODE="<LOCAL>"
DEBU[0028] Output of sudo -S /bin/bash -c docker exec 3fs-storage /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-nodes": Id Type Status Hostname Pid Tags LastHeartbeatTime ConfigVersion ReleaseVersion
1 MGMTD PRIMARY_MGMTD salab-hpedl380g11-02 1 [] N/A 1(UPTODATE) 250228-dev-1-999999-ee9a5cee
100 META HEARTBEAT_CONNECTED salab-hpedl380g11-02 1 [] 2026-06-07 13:28:42 0 250228-dev-1-999999-ee9a5cee NODE="<LOCAL>"
DEBU[0030] Run command: sudo -S /bin/bash -c docker exec 3fs-storage /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-nodes" NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c docker exec 3fs-storage /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-nodes": Id Type Status Hostname Pid Tags LastHeartbeatTime ConfigVersion ReleaseVersion
1 MGMTD PRIMARY_MGMTD salab-hpedl380g11-02 1 [] N/A 1(UPTODATE) 250228-dev-1-999999-ee9a5cee
100 META HEARTBEAT_CONNECTED salab-hpedl380g11-02 1 [] 2026-06-07 13:28:42 0 250228-dev-1-999999-ee9a5cee
10001 STORAGE HEARTBEAT_CONNECTED salab-hpedl380g11-02 1 [] 2026-06-07 13:28:48 1(UPTODATE) 250228-dev-1-999999-ee9a5cee NODE="<LOCAL>"
INFO[0031] Service storage_main is not active NODE=node1 TASK=CreateStorageServiceTask
INFO[0031] Started storage_main container 3fs-storage successfully NODE=node1 TASK=CreateStorageServiceTask
INFO[0031] Creating mount-3fs-disks script and mount-3fs-disks.service service... NODE=node1 TASK=CreateStorageServiceTask
DEBU[0031] Run command: sudo -S /bin/bash -c mktemp -p /tmp NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c mktemp -p /tmp: /tmp/tmp.fh59V7hrQm NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c chmod 0777 /tmp/tmp.fh59V7hrQm NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c chmod 0777 /tmp/tmp.fh59V7hrQm: NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c mkdir -m 0777 -p /mnt/nvme3n1/wayne/opt3fs/bin NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c mkdir -m 0777 -p /mnt/nvme3n1/wayne/opt3fs/bin: NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c cp -r /tmp/tmp.fh59V7hrQm /mnt/nvme3n1/wayne/opt3fs/bin/mount-3fs-disks NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c cp -r /tmp/tmp.fh59V7hrQm /mnt/nvme3n1/wayne/opt3fs/bin/mount-3fs-disks: NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c rm -fr /tmp/tmp.fh59V7hrQm NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c rm -fr /tmp/tmp.fh59V7hrQm: NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c chmod +x /mnt/nvme3n1/wayne/opt3fs/bin/mount-3fs-disks NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c chmod +x /mnt/nvme3n1/wayne/opt3fs/bin/mount-3fs-disks: NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c mktemp -p /tmp NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c mktemp -p /tmp: /tmp/tmp.wHhF42iAaz NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c chmod 0777 /tmp/tmp.wHhF42iAaz NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c chmod 0777 /tmp/tmp.wHhF42iAaz: NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c mkdir -m 0777 -p /lib/systemd/system NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c mkdir -m 0777 -p /lib/systemd/system: NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c cp -r /tmp/tmp.wHhF42iAaz /lib/systemd/system/mount-3fs-disks.service NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c cp -r /tmp/tmp.wHhF42iAaz /lib/systemd/system/mount-3fs-disks.service: NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c rm -fr /tmp/tmp.wHhF42iAaz NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c rm -fr /tmp/tmp.wHhF42iAaz: NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c systemctl enable mount-3fs-disks.service NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c systemctl enable mount-3fs-disks.service: Created symlink /etc/systemd/system/multi-user.target.wants/mount-3fs-disks.service → /usr/lib/systemd/system/mount-3fs-disks.service. NODE="<LOCAL>"
DEBU[0031] Run command: sudo -S /bin/bash -c systemctl daemon-reload NODE="<LOCAL>"
DEBU[0031] Output of sudo -S /bin/bash -c systemctl daemon-reload: NODE="<LOCAL>"
DEBU[0031] Created dir type disk: /mnt/nvme3n1/wayne/opt3fs/storage/3fsdata/data0, index: 0 NODE=node1 TASK=CreateStorageServiceTask
INFO[0031] Running task InitUserAndChainTask
DEBU[0031] Run command: sudo -S /bin/bash -c docker exec 3fs-mgmtd /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml --config.mgmtd_client.mgmtd_server_addresses '["RDMA://10.74.40.86:8000"]' "user-add--root --admin 0 root" NODE="<LOCAL>"
DEBU[0032] Output of sudo -S /bin/bash -c docker exec 3fs-mgmtd /opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml --config.mgmtd_client.mgmtd_server_addresses '["RDMA://10.74.40.86:8000"]' "user-add --root --admin 0 root": Uid 0
Name root
Token AAAUPz+t8QBJAqW/2wB68rDy(Expired at N/A)
IsRootUser true
IsAdmin true
Gid 0
SupplementaryGids NODE="<LOCAL>"
DEBU[0032] Run command: sudo -S /bin/bash -c docker exec 3fs-mgmtd bash -c "echo AAAUPz+t8QBJAqW/2wB68rDy > /opt/3fs/etc/token.txt" NODE="<LOCAL>"
DEBU[0032] Output of sudo -S /bin/bash -c docker exec 3fs-mgmtd bash -c "echo AAAUPz+t8QBJAqW/2wB68rDy > /opt/3fs/etc/token.txt": NODE="<LOCAL>"
DEBU[0032] Run command: sudo -S /bin/bash -c docker exec 3fs-mgmtd python3 /opt/3fs/data_placement/src/model/data_placement.py -ql -relax -type CR --num_nodes 1 --replication_factor 1 --min_targets_per_disk 1 NODE="<LOCAL>"
DEBU[0033] Output of sudo -S /bin/bash -c docker exec 3fs-mgmtd python3 /opt/3fs/data_placement/src/model/data_placement.py -ql -relax -type CR --num_nodes 1 --replication_factor 1 --min_targets_per_disk 1: NODE="<LOCAL>"
DEBU[0033] Command failed stacktrace: exit status 1
Traceback (most recent call last):
File "/opt/3fs/data_placement/src/model/data_placement.py", line 549, in <module>
main()
File "/opt/3fs/data_placement/src/model/data_placement.py", line 515, in main
).run(
File "/opt/3fs/data_placement/src/model/data_placement.py", line 120, in run
num_loops = self.max_recovery_traffic_on_peer*2
File "/opt/3fs/data_placement/src/model/data_placement.py", line 100, in max_recovery_traffic_on_peer
return math.ceil(self.sum_recovery_traffic_per_failure / (self.num_nodes-1))
ZeroDivisionError: division by zero
pkg/external/manager.go:50:(*externalBase).run: sudo run cmd [cmd: docker exec 3fs-mgmtd python3 /opt/3fs/data_placement/src/model/data_placement.py -ql -relax -type CR --num_nodes 1 --replication_factor 1 --min_targets_per_disk 1]
pkg/external/docker.go:168:(*dockerExternal).Exec:
pkg/mgmtd/steps.go:239:(*initUserAndChainStep).initChainFiles: run data_placement.py
pkg/mgmtd/steps.go:177:(*initUserAndChainStep).Execute:
pkg/task/task.go:101:(*BaseTask).ExecuteSteps.(*BaseTask).newStepExecuter.func1:
pkg/task/task.go:139:(*BaseTask).ExecuteSteps:
pkg/task/runner.go:252:(*Runner).Run: run task InitUserAndChainTask
cmd/m3fs/cluster.go:255:createCluster: create cluster
2026/06/07 13:28:53 create cluster: run task InitUserAndChainTask: run data_placement.py: sudo run cmd [cmd: docker exec 3fs-mgmtd python3 /opt/3fs/data_placement/src/model/data_placement.py -ql -relax -typeCR --num_nodes 1 --replication_factor 1 --min_targets_per_disk 1]: exit status 1
Traceback (most recent call last):
File "/opt/3fs/data_placement/src/model/data_placement.py", line 549, in <module>
main()
File "/opt/3fs/data_placement/src/model/data_placement.py", line 515, in main
).run(
File "/opt/3fs/data_placement/src/model/data_placement.py", line 120, in run
num_loops = self.max_recovery_traffic_on_peer*2
File "/opt/3fs/data_placement/src/model/data_placement.py", line 100, in max_recovery_traffic_on_peer
return math.ceil(self.sum_recovery_traffic_per_failure / (self.num_nodes-1))
ZeroDivisionError: division by zero
(3fs) root@salab-hpedl380g11-02:/mnt/nvme0n1/wayne/3fs/m3fs#
team
please see below error