Hadoop 之 压缩Using snappy-java in SnappyCodechttps://issues.apache.org/jira/browse/HADOOP-17125https://github.com/apache/hadoop/pull/2201/files大数据各类引擎关于 Snappy 的使用形式各自相应的jiraparquet-cpp各个依赖项对比Hadoop NativeTask/usr/lib64/libsnappy.so.1/usr/local/hadoop-3.3.3/lib/native/libnativetask.soNativeTask JVM 启动 │ ▼ System.loadLibrary(nativetask)[NativeRuntime.java:53]│ ▼ ld.so 解析 DT_NEEDED → 找 libsnappy.so.1│ ├── 查找顺序:│1.LD_LIBRARY_PATH │2./etc/ld.so.cache由/etc/ld.so.conf*构建 │3.默认路径/usr/lib64,/lib64,/usr/lib,/lib │ ├── ✅ 找到如/usr/lib64/libsnappy.so.1 │ → libnativetask.so 加载成功 │ →supportsCompressionCodec(SnappyCodec)JNI_TRUE │ → spill 压缩走系统 libsnappy 的snappy_compress()│ └── ❌ 找不到 → UnsatisfiedLinkError → NativeRuntime 静态块catchnativeLibraryLoadedfalse → NativeMapOutputCollectorDelegator.init()抛 InvalidJobConfException → createSortingCollector 回退 MapOutputBufferJavaNativeTask 回退机制static{try{System.loadLibrary(nativetask);// ← libnativetask.so 加载LOG.info(Nativetask JNI library loaded.);nativeLibraryLoadedtrue;}catch(finalThrowable t){// Ignore failuresLOG.error(Failed to load nativetask JNI library with error: t);LOG.info(java.library.pathSystem.getProperty(java.library.path));LOG.info(LD_LIBRARY_PATHSystem.getenv(LD_LIBRARY_PATH));// nativeLibraryLoaded 保持 false不抛异常}}publicvoidinit(Context context)throws IOException,ClassNotFoundException{this.contextcontext;this.jobcontext.getJobConf();Platforms.init(job);if(job.getNumReduceTasks()0){String messageThere is no reducer, no need to use native output collector;LOG.error(message);thrownewInvalidJobConfException(message);}Class?comparatorClassjob.getClass(MRJobConfig.KEY_COMPARATOR,null,RawComparator.class);if(comparatorClass!null!Platforms.define(comparatorClass)){String messageNative output collector doesnt support customized java comparator job.get(MRJobConfig.KEY_COMPARATOR);LOG.error(message);thrownewInvalidJobConfException(message);}if(!QuickSort.class.getName().equals(job.get(Constants.MAP_SORT_CLASS))){String messageNative-Task doesnt support sort class job.get(Constants.MAP_SORT_CLASS);LOG.error(message);thrownewInvalidJobConfException(message);}if(job.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY,false)true){String messageNative-Task doesnt support secure shuffle;LOG.error(message);thrownewInvalidJobConfException(message);}finalClass?keyClsjob.getMapOutputKeyClass();try{SuppressWarnings(rawtypes)finalINativeSerializer serializerNativeSerialization.getInstance().getSerializer(keyCls);if(nullserializer){String messageKey type not supported. Cannot find serializer for keyCls.getName();LOG.error(message);thrownewInvalidJobConfException(message);}elseif(!Platforms.support(keyCls.getName(),serializer,job)){String messageNative output collector doesnt support this key, this key is not comparable in native: keyCls.getName();LOG.error(message);thrownewInvalidJobConfException(message);}}catch(finalIOException e){String messageCannot find serializer for keyCls.getName();LOG.error(message);thrownewIOException(message);}finalboolean retNativeRuntime.isNativeLibraryLoaded();if(ret){if(job.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS,false)){String codecjob.get(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC);if(!NativeRuntime.supportsCompressionCodec(codec.getBytes(Charsets.UTF_8))){String messageNative output collector doesnt support compression codec codec;LOG.error(message);thrownewInvalidJobConfException(message);}}NativeRuntime.configure(job);finallongupdateIntervaljob.getLong(Constants.NATIVE_STATUS_UPDATE_INTERVAL,Constants.NATIVE_STATUS_UPDATE_INTERVAL_DEFVAL);updaternewStatusReportChecker(context.getReporter(),updateInterval);updater.start();}else{String messageNativeRuntime cannot be loaded, please check that libnativetask.so is in hadoop library dir;LOG.error(message);thrownewInvalidJobConfException(message);}this.handlernull;try{finalClassKoKClass(ClassK)job.getMapOutputKeyClass();finalClassKoVClass(ClassK)job.getMapOutputValueClass();finalTaskAttemptID idcontext.getMapTask().getTaskID();finalTaskContext taskContextnewTaskContext(job,null,null,oKClass,oVClass,context.getReporter(),id);handlerNativeCollectorOnlyHandler.create(taskContext);}catch(finalIOException e){String messageNative output collector cannot be loaded;;LOG.error(message);thrownewIOException(message,e);}LOG.info(Native output collector can be successfully enabled!);}回退点位出现了hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.javaClass?[]collectorClassesjob.getClasses(JobContext.MAP_OUTPUT_COLLECTOR_CLASS_ATTR,MapOutputBuffer.class);// ↑ 读取 mapreduce.job.map.output.collector.class// 你的配置: NativeMapOutputCollectorDelegator, MapTask$MapOutputBuffer// 逗号分隔 → 数组 [Delegator, MapOutputBuffer]intremainingCollectorscollectorClasses.length;Exception lastExceptionnull;for(Class clazz:collectorClasses){try{// 校验实现接口if(!MapOutputCollector.class.isAssignableFrom(clazz)){throw...;}// 实例化 初始化MapOutputCollectorKEY,VALUEcollectorReflectionUtils.newInstance(subclazz,job);collector.init(context);// ← ①失败就在这里抛LOG.info(Map output collector class collector.getClass().getName());returncollector;// ← 成功即返回}catch(Exception e){String msgUnable to initialize MapOutputCollector clazz.getName();if(--remainingCollectors0){msg (remainingCollectors more collector(s) to try);}lastExceptione;LOG.warn(msg,e);// ← 打 WARN 继续试下一个}}// 全部失败才抛 IOExceptionOS /usr/lib64/libsnappy.so.1#rpm-qi snappyName:snappy Version:1.1.9Release:2.oe2203sp3 Architecture:x86_64 Install Date:Wed17Dec202503:03:47PM CST Group:Unspecified Size:50212License:BSD Signature:RSA/SHA256,Fri05Jan202410:30:58AM CST,Key ID b943e4ef06bfb671 Source RPM:snappy-1.1.9-2.oe2203sp3.src.rpm Build Date:Thu28Dec202301:30:01PM CST Build Host:dc-64g.compass-ci Packager:http://openeuler.orgURL:https://github.com/google/snappySummary:A fast compressor/decompressor Description:Snappy is a compression/decompression library.It doesnotaimformaximum compression,orcompatibility with any other compression library;instead,it aimsforvery high speedsandreasonable compression./usr/local/hadoop-3.3.3/share/hadoop/hdfs/lib/snappy-java-1.1.8.2.jarhttps://github.com/xerial/snappy-javaunzip snappy-java-1.1.8.2.jar-d snappy-java-unzip:/tmp/snappy-java-unzip# find./-name*.so./org/xerial/snappy/native/Linux/x86_64/libsnappyjava.so./org/xerial/snappy/native/Linux/x86/libsnappyjava.so./org/xerial/snappy/native/Linux/android-arm/libsnappyjava.so./org/xerial/snappy/native/Linux/aarch64/libsnappyjava.so./org/xerial/snappy/native/Linux/ppc64le/libsnappyjava.so./org/xerial/snappy/native/Linux/ppc/libsnappyjava.so./org/xerial/snappy/native/Linux/ppc64/libsnappyjava.so./org/xerial/snappy/native/Linux/s390x/libsnappyjava.so./org/xerial/snappy/native/Linux/armv7/libsnappyjava.so./org/xerial/snappy/native/Linux/arm/libsnappyjava.so./org/xerial/snappy/native/Linux/armv6/libsnappyjava.so./org/xerial/snappy/native/FreeBSD/x86_64/libsnappyjava.so./org/xerial/snappy/native/SunOS/sparc/libsnappyjava.so./org/xerial/snappy/native/SunOS/x86_64/libsnappyjava.so./org/xerial/snappy/native/SunOS/x86/libsnappyjava.solibsnappyjava.so 来源不同hadoop version 对 snappy lib 的使用方式snappy-java 跨平台设计完整流程图