MVP. Storage policy support for Ozone

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PipelineRequestInformation.java

@@ -17,17 +17,21 @@
 
 package org.apache.hadoop.hdds.scm;
 
+import org.apache.hadoop.hdds.protocol.StorageType;
+
 /**
  * The information of the request of pipeline.
  */
 public final class PipelineRequestInformation {
   private final long size;
+  private final StorageType storageType;
 
   /**
    * Builder for PipelineRequestInformation.
    */
   public static class Builder {
     private long size;
+    private StorageType storageType;
 
     public static Builder getBuilder() {
       return new Builder();
@@ -43,16 +47,26 @@ public Builder setSize(long sz) {
       return this;
     }
 
+    public Builder setStorageType(StorageType st) {
+      this.storageType = st;
+      return this;
+    }
+
     public PipelineRequestInformation build() {
-      return new PipelineRequestInformation(size);
+      return new PipelineRequestInformation(size, storageType);
     }
   }
 
-  private PipelineRequestInformation(long size) {
+  private PipelineRequestInformation(long size, StorageType storageType) {
     this.size = size;
+    this.storageType = storageType;
   }
 
   public long getSize() {
     return size;
   }
+
+  public StorageType getStorageType() {
+    return storageType;
+  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -445,6 +445,11 @@ public final class ScmConfigKeys {
   public static final String OZONE_SCM_PIPELINE_SCRUB_INTERVAL_DEFAULT =
       "150s";
 
+  public static final String OZONE_SCM_PIPELINE_CREATION_STORAGE_TYPE_AWARE =
+      "ozone.scm.pipeline.creation.storage-type-aware.enabled";
+  public static final boolean
+      OZONE_SCM_PIPELINE_CREATION_STORAGE_TYPE_AWARE_DEFAULT = false;
+
   // Allow SCM to auto create factor ONE ratis pipeline.
   public static final String OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE =
       "ozone.scm.pipeline.creation.auto.factor.one";

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -289,6 +289,7 @@ public final class OzoneConsts {
   public static final String MAX_NUM_OF_BUCKETS = "maxNumOfBuckets";
   public static final String HAS_SNAPSHOT = "hasSnapshot";
   public static final String STORAGE_TYPE = "storageType";
+  public static final String STORAGE_POLICY = "storagePolicy";
   public static final String RESOURCE_TYPE = "resourceType";
   public static final String IS_VERSION_ENABLED = "isVersionEnabled";
   public static final String CREATION_TIME = "creationTime";

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1693,6 +1693,19 @@
       If enabled, SCM will auto create RATIS factor ONE pipeline.
     </description>
   </property>
+  <property>
+    <name>ozone.scm.pipeline.creation.storage-type-aware.enabled</name>
+    <value>false</value>
+    <tag>OZONE, SCM, PIPELINE</tag>
+    <description>
+      If enabled, the background pipeline creator will proactively create
+      storage-type-constrained pipelines for each StorageType (SSD, DISK,
+      ARCHIVE) in addition to untyped pipelines. This ensures that
+      pipelines suitable for HOT/WARM/COLD storage policies are available
+      when containers need to be allocated. Only enable on clusters that
+      use storage tiering with mixed StorageType datanodes.
+    </description>
+  </property>
   <property>
     <name>hdds.scm.safemode.threshold.pct</name>
     <value>0.99</value>
@@ -4195,6 +4208,28 @@
     </description>
   </property>
 
+  <property>
+    <name>ozone.storage.policy.enabled</name>
+    <value>false</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      When enabled, Ozone Manager will resolve and enforce storage policies
+      (HOT, WARM, COLD, ALL_SSD) on buckets and keys. When disabled, all
+      storage policy metadata is ignored and default placement is used.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.default.storage.policy</name>
+    <value>WARM</value>
+    <tag>OZONE, MANAGEMENT</tag>
+    <description>
+      Default storage policy used by Ozone Manager when a client does not
+      specify a storage policy. Supported values are HOT, WARM, COLD,
+      ALL_SSD.
+    </description>
+  </property>
+
   <property>
     <name>ozone.client.max.ec.stripe.write.retries</name>
     <value>10</value>

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java

@@ -23,6 +23,7 @@
 import java.util.concurrent.TimeoutException;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
 import org.apache.hadoop.hdds.scm.AddSCMRequest;
@@ -87,7 +88,7 @@ default List<AllocatedBlock> allocateBlock(long size, int numBlocks,
        ReplicationConfig replicationConfig, String owner,
        ExcludeList excludeList) throws IOException {
     return allocateBlock(size, numBlocks, replicationConfig, owner,
-        excludeList, null);
+        excludeList, null, StorageType.DEFAULT);
   }
 
   /**
@@ -107,9 +108,17 @@ default List<AllocatedBlock> allocateBlock(long size, int numBlocks,
    * @return allocated block accessing info (key, pipeline).
    * @throws IOException
    */
+  default List<AllocatedBlock> allocateBlock(long size, int numBlocks,
+      ReplicationConfig replicationConfig, String owner,
+      ExcludeList excludeList, String clientMachine) throws IOException {
+    return allocateBlock(size, numBlocks, replicationConfig, owner,
+        excludeList, clientMachine, StorageType.DEFAULT);
+  }
+
   List<AllocatedBlock> allocateBlock(long size, int numBlocks,
       ReplicationConfig replicationConfig, String owner,
-      ExcludeList excludeList, String clientMachine) throws IOException;
+      ExcludeList excludeList, String clientMachine,
+      StorageType storageType) throws IOException;
 
   /**
    * Delete blocks for a set of object keys.

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java

@@ -38,6 +38,7 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.conf.StorageUnit;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos.AllocateBlockResponse;
 import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos.AllocateScmBlockRequestProto;
@@ -173,7 +174,8 @@ public List<AllocatedBlock> allocateBlock(
       long size, int num,
       ReplicationConfig replicationConfig,
       String owner, ExcludeList excludeList,
-      String clientMachine
+      String clientMachine,
+      StorageType storageType
   ) throws IOException {
     Preconditions.checkArgument(size > 0, "block size must be greater than 0");
 
@@ -189,6 +191,10 @@ public List<AllocatedBlock> allocateBlock(
       requestBuilder.setClient(clientMachine);
     }
 
+    if (storageType != null) {
+      requestBuilder.setStorageType(storageType.toProto());
+    }
+
     switch (replicationConfig.getReplicationType()) {
     case STAND_ALONE:
       requestBuilder.setFactor(

hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto

@@ -161,6 +161,7 @@ message AllocateScmBlockRequestProto {
 
   optional string client = 9;
 
+  optional hadoop.hdds.StorageTypeProto storageType = 10;
 }
 
 /**

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManager.java

@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.concurrent.TimeoutException;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
 import org.apache.hadoop.ozone.common.BlockGroup;
@@ -42,8 +43,8 @@ public interface BlockManager extends Closeable {
    * @throws IOException
    */
   AllocatedBlock allocateBlock(long size, ReplicationConfig replicationConfig,
-      String owner,
-      ExcludeList excludeList) throws IOException, TimeoutException;
+      String owner, ExcludeList excludeList,
+      StorageType storageType) throws IOException, TimeoutException;
 
   /**
    * Deletes a list of blocks in an atomic operation. Internally, SCM

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java

@@ -32,6 +32,7 @@
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.scm.ScmConfig;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
@@ -145,11 +146,14 @@ public void stop() throws IOException {
   @Override
   public AllocatedBlock allocateBlock(final long size,
       ReplicationConfig replicationConfig,
-      String owner, ExcludeList excludeList)
+      String owner, ExcludeList excludeList,
+      StorageType storageType)
       throws IOException {
     if (LOG.isTraceEnabled()) {
       LOG.trace("Size : {} , replicationConfig: {}", size, replicationConfig);
     }
+    LOG.debug("Allocating block: size={}, replication={}, storageType={}",
+        size, replicationConfig, storageType);
     if (scm.getScmContext().isInSafeMode()) {
       throw new SCMException("SafeModePrecheck failed for allocateBlock",
           SCMException.ResultCodes.SAFE_MODE_EXCEPTION);
@@ -161,7 +165,7 @@ public AllocatedBlock allocateBlock(final long size,
     }
 
     ContainerInfo containerInfo = writableContainerFactory.getContainer(
-        size, replicationConfig, owner, excludeList);
+        size, replicationConfig, owner, excludeList, storageType);
 
     if (containerInfo != null) {
       return newBlock(containerInfo);

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java

@@ -25,11 +25,14 @@
 import static org.apache.hadoop.hdds.scm.ha.SCMService.Event.PRE_CHECK_COMPLETED;
 import static org.apache.hadoop.hdds.scm.ha.SCMService.Event.UNHEALTHY_TO_HEALTHY_NODE_HANDLER_TRIGGERED;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.io.IOException;
 import java.time.Clock;
+import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.Lock;
@@ -40,6 +43,7 @@
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -88,6 +92,7 @@ public class BackgroundPipelineCreator implements SCMService {
   private final AtomicBoolean running = new AtomicBoolean(false);
   private final long intervalInMillis;
   private final Clock clock;
+  private final boolean storageTypeAwareCreation;
 
   BackgroundPipelineCreator(PipelineManager pipelineManager,
       ConfigurationSource conf, SCMContext scmContext, Clock clock) {
@@ -110,6 +115,10 @@ public class BackgroundPipelineCreator implements SCMService {
         ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL_DEFAULT,
         TimeUnit.MILLISECONDS);
 
+    this.storageTypeAwareCreation = conf.getBoolean(
+        ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_STORAGE_TYPE_AWARE,
+        ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_STORAGE_TYPE_AWARE_DEFAULT);
+
     threadName = scmContext.threadNamePrefix() + THREAD_NAME;
   }
 
@@ -203,7 +212,8 @@ private boolean skipCreation(ReplicationConfig replicationConfig,
     return true;
   }
 
-  private void createPipelines() throws RuntimeException {
+  @VisibleForTesting
+  void createPipelines() throws RuntimeException {
     // TODO: #CLUTIL Different replication factor may need to be supported
     HddsProtos.ReplicationType type = HddsProtos.ReplicationType.valueOf(
         conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
@@ -212,8 +222,7 @@ private void createPipelines() throws RuntimeException {
         ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE,
         ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE_DEFAULT);
 
-    List<ReplicationConfig> list =
-        new ArrayList<>();
+    List<ReplicationConfig> replicationConfigs = new ArrayList<>();
     for (HddsProtos.ReplicationFactor factor : HddsProtos.ReplicationFactor
         .values()) {
       if (factor == ReplicationFactor.ZERO) {
@@ -233,10 +242,20 @@ private void createPipelines() throws RuntimeException {
         // Skip this iteration for creating pipeline
         continue;
       }
-      list.add(replicationConfig);
+      replicationConfigs.add(replicationConfig);
+    }
+
+    if (storageTypeAwareCreation) {
+      createTypedPipelines(replicationConfigs);
+    } else {
+      createUntypedPipelines(replicationConfigs);
     }
 
-    LoopingIterator it = new LoopingIterator(list);
+    LOG.debug("BackgroundPipelineCreator createPipelines finished.");
+  }
+
+  private void createUntypedPipelines(List<ReplicationConfig> configs) {
+    LoopingIterator it = new LoopingIterator(configs);
     while (it.hasNext()) {
       ReplicationConfig replicationConfig =
           (ReplicationConfig) it.next();
@@ -251,8 +270,46 @@ private void createPipelines() throws RuntimeException {
         it.remove();
       }
     }
+  }
 
-    LOG.debug("BackgroundPipelineCreator createPipelines finished.");
+  private void createTypedPipelines(List<ReplicationConfig> configs) {
+    // Build (ReplicationConfig, StorageType) pairs: for each config,
+    // one null entry (untyped) plus one per concrete StorageType.
+    StorageType[] storageTypes = {
+        StorageType.SSD, StorageType.DISK, StorageType.ARCHIVE
+    };
+    List<Map.Entry<ReplicationConfig, StorageType>> pairs = new ArrayList<>();
+    for (ReplicationConfig config : configs) {
+      pairs.add(new AbstractMap.SimpleEntry<>(config, null));
+      for (StorageType st : storageTypes) {
+        pairs.add(new AbstractMap.SimpleEntry<>(config, st));
+      }
+    }
+
+    LoopingIterator it = new LoopingIterator(pairs);
+    while (it.hasNext()) {
+      @SuppressWarnings("unchecked")
+      Map.Entry<ReplicationConfig, StorageType> entry =
+          (Map.Entry<ReplicationConfig, StorageType>) it.next();
+
+      try {
+        Pipeline pipeline;
+        if (entry.getValue() == null) {
+          pipeline = pipelineManager.createPipeline(entry.getKey());
+        } else {
+          pipeline = pipelineManager.createPipeline(
+              entry.getKey(), entry.getValue());
+        }
+        LOG.info("Created new pipeline {} with StorageType {}",
+            pipeline, entry.getValue());
+      } catch (IOException ioe) {
+        it.remove();
+      } catch (Throwable t) {
+        LOG.error("Error while creating pipelines for StorageType "
+            + entry.getValue(), t);
+        it.remove();
+      }
+    }
   }
 
   @Override

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java

@@ -25,6 +25,7 @@
 import java.util.Set;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.ContainerReplica;
 import org.apache.hadoop.hdds.utils.db.CodecException;
@@ -39,6 +40,11 @@ public interface PipelineManager extends Closeable, PipelineManagerMXBean {
   Pipeline createPipeline(ReplicationConfig replicationConfig)
       throws IOException;
 
+  default Pipeline createPipeline(ReplicationConfig replicationConfig,
+      StorageType storageType) throws IOException {
+    return createPipeline(replicationConfig);
+  }
+
   Pipeline createPipeline(ReplicationConfig replicationConfig,
                           List<DatanodeDetails> excludedNodes,
                           List<DatanodeDetails> favoredNodes)