The WolfspyreLabs Blog/ 2023/ Posts from July/ More fun with Ceph RADOSGW/ Improving RadosGW Performance/ Improving RadosGW Performance sysctl #https://www.petasan.org/forums/?view=thread&id=63 https://sysctl-explorer.net/net/ipv4/accept_redirects/ net.ipv4.conf.all.accept_redirects = 0 net.ipv4.conf.all.secure_redirects = 0 net.ipv4.conf.default.secure_redirects = 0 https://sysctl-explorer.net/net/ipv4/secure_redirects/ net.ipv4.conf.all.send_redirects = 0 net.ipv4.conf.all.accept_source_route = 0 net.ipv4.conf.default.accept_source_route = 0 net.ipv4.conf.lo.accept_source_route = 0 https://sysctl-explorer.net/net/ipv4/rp_filter/ net.ipv4.conf.all.rp_filter = 1 net.ipv4.conf.default.rp_filter = 1 net.ipv4.conf.lo.rp_filter = 1 net.ipv4.conf.all.arp_filter = 1 net.ipv4.conf.default.arp_filter = 1 net.ipv4.icmp_echo_ignore_broadcasts = 1 https://www.tenable.com/audits/items/CIS_Distribution_Independent_Linux_Server_L1_v2.0.0.audit:0bd887213fb6fe5c4159a04e7278af2e net.ipv4.icmp_ignore_bogus_error_responses = 1 https://www.tenable.com/audits/items/CIS_Distribution_Independent_Linux_Server_L1_v2.0.0.audit:4fbfc14df18d3e1dba83456abc1e898f net.ipv4.tcp_rfc1337 = 1 net.ipv4.tcp_syncookies = 0 https://www.geeksforgeeks.org/how-syn-cookies-are-used-to-preventing-syn-flood-attack/ kernel.sysrq = 0 https://www.kernel.org/doc/html/latest/admin-guide/sysrq.html #vm.swappiness = 0 https://sysctl-explorer.net/vm/swappiness/ vm.vfs_cache_pressure = 50 https://sysctl-explorer.net/vm/vfs_cache_pressure/ vm.min_free_kbytes = 262144 vm.min_slab_ratio = 1 https://sysctl-explorer.net/vm/min_slab_ratio/ vm.dirty_background_ratio = 5 https://sysctl-explorer.net/vm/dirty_background_ratio/ vm.dirty_ratio = 10 https://sysctl-explorer.net/vm/dirty_ratio/ vm.dirty_writeback_centisecs = 3000 https://sysctl-explorer.net/vm/dirty_writeback_centisecs/ vm.dirty_expire_centisecs = 9000 https://sysctl-explorer.net/vm/dirty_expire_centisecs/ vm.zone_reclaim_mode = 0 https://sysctl-explorer.net/vm/zone_reclaim_mode/ #fs.file-max = 78718144 fs.file-max = 26234859 https://sysctl-explorer.net/fs/filemax%20&%20filenr/ fs.inotify.max_user_watches = 16777216 fs.inotify.max_queued_events = 65536 kernel.pid_max = 4194303 https://sysctl-explorer.net/kernel/pid_max/ kernel.core_uses_pid = 1 https://sysctl-explorer.net/kernel/core_uses_pid/ kernel.panic_on_oops = 1 https://sysctl-explorer.net/kernel/panic_on_oops/ kernel.panic = 60 https://sysctl-explorer.net/kernel/panic/ #net.core.rmem_max = 268435456 https://sysctl-explorer.net/net/core/rmem_max/ net.core.wmem_max = 268435456 67108864 net.core.rmem_default = 67108864 https://sysctl-explorer.net/net/core/rmem_default/ net.core.wmem_default = 67108864 https://sysctl-explorer.net/net/core/wmem_default/ net.core.optmem_max = 134217728 https://sysctl-explorer.net/net/core/optmem_max/ net.core.netdev_budget = 1200 https://sysctl-explorer.net/net/core/netdev_budget/ net.core.netdev_max_backlog = 300000 https://sysctl-explorer.net/net/core/netdev_max_backlog/ #net.core.somaxconn = 65536 net.core.somaxconn = 32768 https://sysctl-explorer.net/net/ipv4/somaxconn/ net.ipv4.ip_local_port_range = 1024 65000 net.ipv4.udp_rmem_min = 131072 https://sysctl-explorer.net/net/ipv4/udp_rmem_min/ net.ipv4.udp_wmem_min = 131072 https://sysctl-explorer.net/net/ipv4/udp_wmem_min/ net.ipv4.tcp_moderate_rcvbuf = 1 https://sysctl-explorer.net/net/ipv4/tcp_moderate_rcvbuf/ net.ipv4.tcp_mtu_probing = 1 https://sysctl-explorer.net/net/ipv4/tcp_mtu_probing/ net.ipv4.tcp_low_latency = 1 https://sysctl-explorer.net/net/ipv4/tcp_low_latency/ net.ipv4.tcp_adv_win_scale = 1 https://sysctl-explorer.net/net/ipv4/tcp_adv_win_scale/ net.ipv4.tcp_max_syn_backlog = 819200 https://sysctl-explorer.net/net/ipv4/tcp_max_syn_backlog/ net.ipv4.tcp_synack_retries = 1 https://sysctl-explorer.net/net/ipv4/tcp_synack_retries/ net.ipv4.tcp_max_tw_buckets = 2000000 https://sysctl-explorer.net/net/ipv4/tcp_max_tw_buckets/ net.ipv4.tcp_max_orphans = 65536 net.ipv4.tcp_orphan_retries = 0 net.ipv4.tcp_fin_timeout = 10 https://sysctl-explorer.net/net/ipv4/tcp_fin_timeout/ net.ipv4.tcp_keepalive_time = 1800 https://sysctl-explorer.net/net/ipv4/tcp_keepalive_time/ net.ipv4.tcp_keepalive_intvl = 10 https://sysctl-explorer.net/net/ipv4/tcp_keepalive_intvl/ net.ipv4.tcp_keepalive_probes = 5 https://sysctl-explorer.net/net/ipv4/tcp_keepalive_probes/ net.ipv4.tcp_timestamps = 1 https://sysctl-explorer.net/net/ipv4/tcp_timestamps/ net.ipv4.tcp_sack = 1 https://sysctl-explorer.net/net/ipv4/tcp_sack/ net.ipv4.tcp_tw_reuse = 1 https://sysctl-explorer.net/net/ipv4/tcp_tw_reuse/ net.ipv4.tcp_rmem = 67108864 134217728 268435456 net.ipv4.tcp_wmem = 67108864 134217728 268435456 net.ipv4.tcp_notsent_lowat = 4294967295 net.ipv4.tcp_window_scaling = 1 https://sysctl-explorer.net/net/ipv4/tcp_window_scaling/ net.ipv4.tcp_congestion_control = htcp net.ipv4.tcp_no_metrics_save = 1 net.ipv4.tcp_slow_start_after_idle = 0 #net.netfilter.nf_conntrack_max = 16777216 #net.netfilter.nf_conntrack_tcp_timeout_established = 1800 kernel.nmi_watchdog = 0 https://www.cyberciti.biz/faq/gentoo-centos-rhel-debian-fedora-increasing-txqueuelen/ https://www.pc-freak.net/blog/why-and-how-to-increase-and-decrease-txqueuelen-transmit-queue-lenght-in-gnu-linux/ Old Intel sponsored research on performance of ceph #VM Read-ahead #echo 2048 > /sys/block/${disk}/queue/read_ahead_kb (default 128) for block in `ls /sys/block`; do echo 2048 > /sys/block/${block}/queue/read_ahead_kb ;done CPU Pinning # #[global] fsid = 35b08d01-b688-4b9a-947b-bc2e25719370 mon_initial_members = gw2 mon_host = 10.10.10.105 filestore_xattr_use_omap = true auth_cluster_required = none auth_service_required = none auth_client_required = none debug_lockdep = 0/0 debug_context = 0/0 debug_crush = 0/0 debug_buffer = 0/0 debug_timer = 0/0 debug_filer = 0/0 debug_objecter = 0/0 debug_rados = 0/0 debug_rbd = 0/0 debug_journaler = 0/0 debug_objectcatcher = 0/0 debug_client = 0/0 debug_osd = 0/0 debug_optracker = 0/0 debug_objclass = 0/0 debug_filestore = 0/0 debug_journal = 0/0 debug_ms = 0/0 debug_monc = 0/0 debug_tp = 0/0 debug_auth = 0/0 debug_finisher = 0/0 debug_heartbeatmap = 0/0 debug_perfcounter = 0/0 debug_asok = 0/0 debug_throttle = 0/0 debug_mon = 0/0 debug_paxos = 0/0 debug_rgw = 0/0 [mon] mon_pg_warn_max_per_osd=5000 mon_max_pool_pg_num=106496 [client] rbd cache = false [osd] osd mkfs type = xfs osd mount options xfs = rw,noatime,,nodiratime,inode64,logbsize=256k,delaylog osd mkfs options xfs = -f -i size=2048 filestore_queue_max_ops=5000 filestore_queue_max_bytes = 1048576000 filestore_max_sync_interval = 10 filestore_merge_threshold = 500 filestore_split_multiple = 100 osd_op_shard_threads = 8 journal_max_write_entries = 5000 journal_max_write_bytes = 1048576000 journal_queueu_max_ops = 3000 journal_queue_max_bytes = 1048576000 ms_dispatch_throttle_bytes = 1048576000 objecter_inflight_op_bytes = 1048576000 public_network = 10.10.10.100/24 cluster_network = 10.10.10.100/24 [client.radosgw.gw2-1] host = gw2 keyring = /etc/ceph/ceph.client.radosgw.keyring rgw cache enabled = true rgw cache lru size = 100000 rgw socket path = /var/run/ceph/ceph.client.radosgw.gw2-1.fastcgi.sock rgw thread pool size = 256 rgw enable ops log = false rgw enable usage log = false log file = /dev/null rgw frontends =civetweb port=80 rgw override bucket index max shards = 8 https://tracker.ceph.com/projects/ceph/wiki/Tuning_for_All_Flash_Deployments