Environment

Runtime Information

NameValue
Java Home/usr/lib/jvm/sapmachine-jre
Java Version17.0.16 (SAP SE)
Scala Versionversion 2.12.18

Spark Properties

NameValue
spark.app.idspark-eb802a75ad4649d5828265b44d51d59d
spark.app.initial.jar.urls*********(redacted)
spark.app.namecrp-dl-stream
spark.app.startTime1773806317233
spark.app.submitTime1773806317129
spark.authenticatetrue
spark.blockManager.port7079
spark.checkpoint.compresstrue
spark.cleaner.referenceTracking.cleanCheckpointstrue
spark.databricks.delta.merge.repartitionBeforeWrite.enabledtrue
spark.databricks.delta.schema.autoMerge.enabledtrue
spark.delta.logStore.hdlfs.implcom.sap.hana.datalake.files.HdlfsLogStore
spark.driver.extraClassPath/app/dependant-libs/guava-32.1.3-jre.jar:/app/dependant-libs/protobuf-java-3.24.3.jar:/app/dependant-libs/failureaccess-1.0.1.jar:/app/dependant-libs/netty-codec-4.2.8.Final.jar:/app/dependant-libs/json-20231013.jar
spark.driver.extraJavaOptions-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false -XX:MaxDirectMemorySize=1G -Dlog4j2.configurationFile=/etc/configuration/log4j2.properties
spark.driver.host10.96.14.18
spark.driver.memory10g
spark.driver.memoryOverhead3g
spark.driver.port7078
spark.driverEnv.MALLOC_ARENA_MAX2
spark.dynamicAllocation.cachedExecutorIdleTimeout900s
spark.dynamicAllocation.enabledtrue
spark.dynamicAllocation.executorAllocationRatio0.25
spark.dynamicAllocation.executorIdleTimeout300s
spark.dynamicAllocation.maxExecutors20
spark.dynamicAllocation.minExecutors2
spark.dynamicAllocation.schedulerBacklogTimeout60s
spark.dynamicAllocation.shuffleTracking.enabledtrue
spark.dynamicAllocation.shuffleTracking.timeout900s
spark.executor.cores3
spark.executor.extraClassPath/app/dependant-libs/guava-32.1.3-jre.jar:/app/dependant-libs/protobuf-java-3.24.3.jar:/app/dependant-libs/failureaccess-1.0.1.jar:/app/dependant-libs/netty-codec-4.2.8.Final.jar:/app/dependant-libs/json-20231013.jar
spark.executor.extraJavaOptions-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false -XX:MaxDirectMemorySize=1G -Dlog4j2.configurationFile=/etc/configuration/log4j2.properties
spark.executor.iddriver
spark.executor.instances2
spark.executor.memory6g
spark.executor.memoryOverhead6g
spark.executorEnv.MALLOC_ARENA_MAX2
spark.executorEnv.RUNTIME_NAMESPACEstage
spark.executorEnv.grafana_domaingrafana.staging.predictivereplenishment.cloud.sap
spark.hadoop.fs.AbstractFileSystem.hdlfs.implcom.sap.hana.datalake.files.Hdlfs
spark.hadoop.fs.azure.account.auth.type.ciccsstagingeudisk.dfs.core.windows.netOAuth
spark.hadoop.fs.azure.account.auth.type.stdfsstageeu.dfs.core.windows.netOAuth
spark.hadoop.fs.azure.account.oauth.provider.type.ciccsstagingeudisk.dfs.core.windows.net*********(redacted)
spark.hadoop.fs.azure.account.oauth.provider.type.stdfsstageeu.dfs.core.windows.net*********(redacted)
spark.hadoop.fs.azure.account.oauth2.client.endpoint.ciccsstagingeudisk.dfs.core.windows.net*********(redacted)
spark.hadoop.fs.azure.account.oauth2.client.endpoint.stdfsstageeu.dfs.core.windows.net*********(redacted)
spark.hadoop.fs.azure.account.oauth2.client.id.ciccsstagingeudisk.dfs.core.windows.net84611f45-729d-4df8-9cf6-c505ea3267b3
spark.hadoop.fs.azure.account.oauth2.client.id.stdfsstageeu.dfs.core.windows.net84611f45-729d-4df8-9cf6-c505ea3267b3
spark.hadoop.fs.azure.account.oauth2.client.secret.ciccsstagingeudisk.dfs.core.windows.net*********(redacted)
spark.hadoop.fs.azure.account.oauth2.client.secret.stdfsstageeu.dfs.core.windows.net*********(redacted)
spark.hadoop.fs.hdlfs.direct-access.http-client.connection-pool.event-interceptor.factory.classcom.sap.hana.datalake.files.utils.http.ConnectionPoolEventLoggerFactory
spark.hadoop.fs.hdlfs.direct-access.http-client.connections.max-count130
spark.hadoop.fs.hdlfs.direct-access.modeEXPECT
spark.hadoop.fs.hdlfs.direct-access.multipart-upload.max-active-chunk2
spark.hadoop.fs.hdlfs.implcom.sap.hana.datalake.files.HdlfsFileSystem
spark.hadoop.fs.hdlfs.impl.disable.cachefalse
spark.hadoop.fs.hdlfs.ssl.keystores.factory.classcom.sap.s4hana.eureka.business.crpdlstream.hdlfs.HdlfsTenantKeyStoresFactory
spark.io.encryption.enabledtrue
spark.jars*********(redacted)
spark.kubernetes.allocation.driver.readinessTimeout60000
spark.kubernetes.allocation.executor.timeout60000
spark.kubernetes.authenticate.driver.serviceAccountNamecrp-dl-stream
spark.kubernetes.container.imagedeploy-releases-hyperspace-docker.common.repositories.cloud.sap/crp-dl-stream-service:LATEST-20260317070323-a8a1d7595a02cc186183ae0396284b7cef563022
spark.kubernetes.container.image.pullSecrets*********(redacted)
spark.kubernetes.driver.connectionTimeout60000
spark.kubernetes.driver.container.imagedeploy-releases-hyperspace-docker.common.repositories.cloud.sap/crp-dl-stream-service:LATEST-20260317070323-a8a1d7595a02cc186183ae0396284b7cef563022
spark.kubernetes.driver.label.appcrp-dl-stream-stream
spark.kubernetes.driver.label.spark_appcrp-dl-stream-stream
spark.kubernetes.driver.limit.cores3
spark.kubernetes.driver.limit.memory10g
spark.kubernetes.driver.pod.namecrp-dl-stream-b67f8b7d9-ldh7v
spark.kubernetes.driver.request.cores3
spark.kubernetes.driver.requestTimeout60000
spark.kubernetes.driver.requests.memory10g
spark.kubernetes.driverEnv.RUNTIME_NAMESPACEstage
spark.kubernetes.driverEnv.grafana_domaingrafana.staging.predictivereplenishment.cloud.sap
spark.kubernetes.executor.container.imagedeploy-releases-hyperspace-docker.common.repositories.cloud.sap/crp-dl-stream-service:LATEST-20260317070323-a8a1d7595a02cc186183ae0396284b7cef563022
spark.kubernetes.executor.deleteOnTerminationtrue
spark.kubernetes.executor.label.appcrp-dl-stream-stream
spark.kubernetes.executor.label.spark_appcrp-dl-stream-stream
spark.kubernetes.executor.label.spark_driver_podcrp-dl-stream-b67f8b7d9-ldh7v
spark.kubernetes.executor.limit.cores3
spark.kubernetes.executor.limit.memory6g
spark.kubernetes.executor.podNamePrefixcrp-dl-stream-b67f8b7d9-ldh7v
spark.kubernetes.executor.request.cores3
spark.kubernetes.executor.requests.memory6g
spark.kubernetes.file.upload.path/tmp
spark.kubernetes.local.dirs.tmpfstrue
spark.kubernetes.memoryOverheadFactor0.4
spark.kubernetes.namespacestage
spark.kubernetes.report.interval30000
spark.kubernetes.submission.waitAppCompletiontrue
spark.local.dir/cache
spark.masterk8s://https://api.e923380.kyma.internal.live.k8s.ondemand.com:443
spark.memory.offHeap.enabledtrue
spark.memory.offHeap.size2g
spark.network.crypto.enabledtrue
spark.rdd.compresstrue
spark.repl.local.jars*********(redacted)
spark.scheduler.allocation.file/etc/configuration/fairscheduler.xml
spark.scheduler.modeFAIR
spark.serializerorg.apache.spark.serializer.KryoSerializer
spark.sql.autoBroadcastJoinThreshold524288
spark.sql.catalog.spark_catalogorg.apache.spark.sql.delta.catalog.DeltaCatalog
spark.sql.extensionsio.delta.sql.DeltaSparkSessionExtension
spark.sql.legacy.parquet.int96RebaseModeInReadLEGACY
spark.sql.parquet.outputTimestampTypeTIMESTAMP_MICROS
spark.sql.redaction.options.regex(fs\.hdlfs\.ssl\.(key|cert)file)
spark.sql.session.timeZoneUTC
spark.submit.deployModeclient
spark.submit.pyFiles
spark.ui.retainedJobs5000
spark.ui.retainedStages5000
spark.ui.strictTransportSecuritymax-age=31536000

Resource Profiles

Resource Profile IdResource Profile Contents
0
Executor Reqs:
	memoryOverhead: [amount: 6144]
	cores: [amount: 3]
	memory: [amount: 6144]
	offHeap: [amount: 2048]
Task Reqs:
	cpus: [amount: 1.0]

Hadoop Properties

System Properties

Metrics Properties

Classpath Entries