[T1-1-2] S-hhhhh

include/infiniop.h

@@ -17,5 +17,14 @@
 #include "infiniop/ops/sub.h"
 #include "infiniop/ops/swiglu.h"
 #include "infiniop/tensor_descriptor.h"
+#include "infiniop/ops/div.h"
+#include "infiniop/ops/silu.h"
+#include "infiniop/ops/and.h"
+#include "infiniop/ops/or.h"
+#include "infiniop/ops/equal.h"
+#include "infiniop/ops/relu_backward.h"
+#include "infiniop/ops/gelu.h"
+#include "infiniop/ops/gelu_backward.h"
+#include "infiniop/ops/cross_entropy_loss_backward.h"
 
 #endif // __INFINIOP_API_H__

include/infiniop/ops/and.h

@@ -0,0 +1,26 @@
+#ifndef __INFINIOP_AND_API_H__
+#define __INFINIOP_AND_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopAndDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateAndDescriptor(infiniopHandle_t handle,
+                                                        infiniopAndDescriptor_t *desc_ptr,
+                                                        infiniopTensorDescriptor_t c,
+                                                        infiniopTensorDescriptor_t a,
+                                                        infiniopTensorDescriptor_t b);
+
+__C __export infiniStatus_t infiniopGetAndWorkspaceSize(infiniopAndDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopAnd(infiniopAndDescriptor_t desc,
+                                        void *workspace,
+                                        size_t workspace_size,
+                                        void *c,
+                                        const void *a,
+                                        const void *b,
+                                        void *stream);
+
+__C __export infiniStatus_t infiniopDestroyAndDescriptor(infiniopAndDescriptor_t desc);
+
+#endif

include/infiniop/ops/cross_entropy_loss_backward.h

@@ -0,0 +1,31 @@
+#ifndef __INFINIOP_CROSS_ENTROPY_LOSS_BACKWARD_API_H__
+#define __INFINIOP_CROSS_ENTROPY_LOSS_BACKWARD_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopCrossEntropyLossBackwardDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateCrossEntropyLossBackwardDescriptor(
+    infiniopHandle_t handle,
+    infiniopCrossEntropyLossBackwardDescriptor_t *desc_ptr,
+    infiniopTensorDescriptor_t grad_logits_desc,
+    infiniopTensorDescriptor_t probs_desc,
+    infiniopTensorDescriptor_t target_desc);
+
+__C __export infiniStatus_t infiniopGetCrossEntropyLossBackwardWorkspaceSize(
+    infiniopCrossEntropyLossBackwardDescriptor_t desc, 
+    size_t *size);
+
+__C __export infiniStatus_t infiniopCrossEntropyLossBackward(
+    infiniopCrossEntropyLossBackwardDescriptor_t desc,
+    void *workspace,
+    size_t workspace_size,
+    void *grad_logits,
+    const void *probs,
+    const void *target,
+    void *stream);
+
+__C __export infiniStatus_t infiniopDestroyCrossEntropyLossBackwardDescriptor(
+    infiniopCrossEntropyLossBackwardDescriptor_t desc);
+
+#endif

include/infiniop/ops/div.h

@@ -0,0 +1,26 @@
+#ifndef __INFINIOP_DIV_API_H__
+#define __INFINIOP_DIV_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopDivDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateDivDescriptor(infiniopHandle_t handle,
+                                                        infiniopDivDescriptor_t *desc_ptr,
+                                                        infiniopTensorDescriptor_t c,
+                                                        infiniopTensorDescriptor_t a,
+                                                        infiniopTensorDescriptor_t b);
+
+__C __export infiniStatus_t infiniopGetDivWorkspaceSize(infiniopDivDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopDiv(infiniopDivDescriptor_t desc,
+                                        void *workspace,
+                                        size_t workspace_size,
+                                        void *c,
+                                        const void *a,
+                                        const void *b,
+                                        void *stream);
+
+__C __export infiniStatus_t infiniopDestroyDivDescriptor(infiniopDivDescriptor_t desc);
+
+#endif

include/infiniop/ops/equal.h

@@ -0,0 +1,26 @@
+#ifndef __INFINIOP_EQUAL_API_H__
+#define __INFINIOP_EQUAL_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopEqualDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateEqualDescriptor(infiniopHandle_t handle,
+                                                          infiniopEqualDescriptor_t *desc_ptr,
+                                                          infiniopTensorDescriptor_t c,
+                                                          infiniopTensorDescriptor_t a,
+                                                          infiniopTensorDescriptor_t b);
+
+__C __export infiniStatus_t infiniopGetEqualWorkspaceSize(infiniopEqualDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopEqual(infiniopEqualDescriptor_t desc,
+                                          void *workspace,
+                                          size_t workspace_size,
+                                          void *c,
+                                          const void *a,
+                                          const void *b,
+                                          void *stream);
+
+__C __export infiniStatus_t infiniopDestroyEqualDescriptor(infiniopEqualDescriptor_t desc);
+
+#endif

include/infiniop/ops/gelu.h

@@ -0,0 +1,24 @@
+#ifndef __INFINIOP_GELU_API_H__
+#define __INFINIOP_GELU_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopGeluDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateGeluDescriptor(infiniopHandle_t handle,
+                                                         infiniopGeluDescriptor_t *desc_ptr,
+                                                         infiniopTensorDescriptor_t output,
+                                                         infiniopTensorDescriptor_t input);
+
+__C __export infiniStatus_t infiniopGetGeluWorkspaceSize(infiniopGeluDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopGelu(infiniopGeluDescriptor_t desc,
+                                         void *workspace,
+                                         size_t workspace_size,
+                                         void *output,
+                                         const void *input,
+                                         void *stream);
+
+__C __export infiniStatus_t infiniopDestroyGeluDescriptor(infiniopGeluDescriptor_t desc);
+
+#endif

include/infiniop/ops/gelu_backward.h

@@ -0,0 +1,26 @@
+#ifndef __INFINIOP_GELU_BACKWARD_API_H__
+#define __INFINIOP_GELU_BACKWARD_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopGeluBackwardDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateGeluBackwardDescriptor(infiniopHandle_t handle,
+                                                                infiniopGeluBackwardDescriptor_t *desc_ptr,
+                                                                infiniopTensorDescriptor_t grad_input,
+                                                                infiniopTensorDescriptor_t input,
+                                                                infiniopTensorDescriptor_t grad_output);
+
+__C __export infiniStatus_t infiniopGetGeluBackwardWorkspaceSize(infiniopGeluBackwardDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopGeluBackward(infiniopGeluBackwardDescriptor_t desc,
+                                                void *workspace,
+                                                size_t workspace_size,
+                                                void *grad_input,
+                                                const void *input,
+                                                const void *grad_output,
+                                                void *stream);
+
+__C __export infiniStatus_t infiniopDestroyGeluBackwardDescriptor(infiniopGeluBackwardDescriptor_t desc);
+
+#endif

include/infiniop/ops/or.h

@@ -0,0 +1,26 @@
+#ifndef __INFINIOP_OR_API_H__
+#define __INFINIOP_OR_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopOrDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateOrDescriptor(infiniopHandle_t handle,
+                                                        infiniopOrDescriptor_t *desc_ptr,
+                                                        infiniopTensorDescriptor_t c,
+                                                        infiniopTensorDescriptor_t a,
+                                                        infiniopTensorDescriptor_t b);
+
+__C __export infiniStatus_t infiniopGetOrWorkspaceSize(infiniopOrDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopOr(infiniopOrDescriptor_t desc,
+                                        void *workspace,
+                                        size_t workspace_size,
+                                        void *c,
+                                        const void *a,
+                                        const void *b,
+                                        void *stream);
+
+__C __export infiniStatus_t infiniopDestroyOrDescriptor(infiniopOrDescriptor_t desc);
+
+#endif

include/infiniop/ops/relu_backward.h

@@ -0,0 +1,26 @@
+#ifndef __INFINIOP_RELU_BACKWARD_API_H__
+#define __INFINIOP_RELU_BACKWARD_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopReluBackwardDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateReluBackwardDescriptor(infiniopHandle_t handle,
+                                                                 infiniopReluBackwardDescriptor_t *desc_ptr,
+                                                                 infiniopTensorDescriptor_t grad_input,
+                                                                 infiniopTensorDescriptor_t input,
+                                                                 infiniopTensorDescriptor_t grad_output);
+
+__C __export infiniStatus_t infiniopGetReluBackwardWorkspaceSize(infiniopReluBackwardDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopReluBackward(infiniopReluBackwardDescriptor_t desc,
+                                                 void *workspace,
+                                                 size_t workspace_size,
+                                                 void *grad_input,
+                                                 const void *input,
+                                                 const void *grad_output,
+                                                 void *stream);
+
+__C __export infiniStatus_t infiniopDestroyReluBackwardDescriptor(infiniopReluBackwardDescriptor_t desc);
+
+#endif

include/infiniop/ops/silu.h

@@ -0,0 +1,24 @@
+#ifndef __INFINIOP_SILU_API_H__
+#define __INFINIOP_SILU_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopSiluDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateSiluDescriptor(infiniopHandle_t handle,
+                                                         infiniopSiluDescriptor_t *desc_ptr,
+                                                         infiniopTensorDescriptor_t output,
+                                                         infiniopTensorDescriptor_t input);
+
+__C __export infiniStatus_t infiniopGetSiluWorkspaceSize(infiniopSiluDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopSilu(infiniopSiluDescriptor_t desc,
+                                         void *workspace,
+                                         size_t workspace_size,
+                                         void *output,
+                                         const void *input,
+                                         void *stream);
+
+__C __export infiniStatus_t infiniopDestroySiluDescriptor(infiniopSiluDescriptor_t desc);
+
+#endif

scripts/python_test.py

@@ -24,6 +24,15 @@ def run_tests(args):
         "rope.py",
         "sub.py",
         "swiglu.py",
+        "div.py",
+        "silu.py",
+        "and.py",
+        "or.py",
+        "equal.py",
+        "relu_backward.py",
+        "gelu.py",
+        "gelu_backward.py",
+        "cross_entropy_loss_backward.py",
     ]:
         result = subprocess.run(
             f"python {test} {args} --debug", text=True, encoding="utf-8", shell=True

src/infiniop-test/include/ops.hpp

@@ -16,6 +16,15 @@ DECLARE_INFINIOP_TEST(add)
 DECLARE_INFINIOP_TEST(causal_softmax)
 DECLARE_INFINIOP_TEST(rearrange)
 DECLARE_INFINIOP_TEST(sub)
+DECLARE_INFINIOP_TEST(div)
+DECLARE_INFINIOP_TEST(silu)
+DECLARE_INFINIOP_TEST(logical_and)
+DECLARE_INFINIOP_TEST(logical_or)
+DECLARE_INFINIOP_TEST(equal)
+DECLARE_INFINIOP_TEST(relu_backward)
+DECLARE_INFINIOP_TEST(gelu)
+DECLARE_INFINIOP_TEST(gelu_backward)
+DECLARE_INFINIOP_TEST(cross_entropy_loss_backward)
 
 #define REGISTER_INFINIOP_TEST(name)                      \
     {                                                     \
@@ -30,19 +39,28 @@ DECLARE_INFINIOP_TEST(sub)
 /*
  * Register all the tests here
  */
-#define TEST_BUILDER_MAPPINGS                  \
-    {                                          \
-        REGISTER_INFINIOP_TEST(gemm)           \
-        REGISTER_INFINIOP_TEST(random_sample)  \
-        REGISTER_INFINIOP_TEST(add)            \
-        REGISTER_INFINIOP_TEST(mul)            \
-        REGISTER_INFINIOP_TEST(clip)           \
-        REGISTER_INFINIOP_TEST(swiglu)         \
-        REGISTER_INFINIOP_TEST(rope)           \
-        REGISTER_INFINIOP_TEST(rms_norm)       \
-        REGISTER_INFINIOP_TEST(causal_softmax) \
-        REGISTER_INFINIOP_TEST(rearrange)      \
-        REGISTER_INFINIOP_TEST(sub)            \
+#define TEST_BUILDER_MAPPINGS                               \
+    {                                                       \
+        REGISTER_INFINIOP_TEST(gemm)                        \
+        REGISTER_INFINIOP_TEST(random_sample)               \
+        REGISTER_INFINIOP_TEST(add)                         \
+        REGISTER_INFINIOP_TEST(mul)                         \
+        REGISTER_INFINIOP_TEST(clip)                        \
+        REGISTER_INFINIOP_TEST(swiglu)                      \
+        REGISTER_INFINIOP_TEST(rope)                        \
+        REGISTER_INFINIOP_TEST(rms_norm)                    \
+        REGISTER_INFINIOP_TEST(causal_softmax)              \
+        REGISTER_INFINIOP_TEST(rearrange)                   \
+        REGISTER_INFINIOP_TEST(sub)                         \
+        REGISTER_INFINIOP_TEST(div)                         \
+        REGISTER_INFINIOP_TEST(silu)                        \
+        REGISTER_INFINIOP_TEST(logical_and)                 \
+        REGISTER_INFINIOP_TEST(logical_or)                  \
+        REGISTER_INFINIOP_TEST(equal)                       \
+        REGISTER_INFINIOP_TEST(relu_backward)               \
+        REGISTER_INFINIOP_TEST(gelu)                        \
+        REGISTER_INFINIOP_TEST(gelu_backward)               \
+        REGISTER_INFINIOP_TEST(cross_entropy_loss_backward) \
     }
 
 namespace infiniop_test {

src/infiniop-test/include/test.hpp

@@ -47,18 +47,19 @@ std::vector<std::shared_ptr<Result>> runAllTests(
     const GGUFFileReader &,
     infiniDevice_t device, int device_id,
     size_t warm_ups, size_t iterations,
-    double rtol, double atol);
+    double rtol, double atol, bool equal_nan = false);
 
 // Run a single test read from a GGUF file
 std::shared_ptr<Result> runTest(
     const GGUFFileReader &,
     infiniDevice_t device, int device_id,
     size_t warm_ups, size_t iterations,
     double rtol, double atol,
-    size_t test_id);
+    size_t test_id,
+    bool equal_nan = false);
 
 // Check if two tensors are close within given tolerance
-void allClose(std::shared_ptr<Tensor> actual, std::shared_ptr<Tensor> expected, double rtol = 1e-3, double atol = 1e-3);
+void allClose(std::shared_ptr<Tensor> actual, std::shared_ptr<Tensor> expected, double rtol = 1e-3, double atol = 1e-3, bool equal_nan = false);
 
 // Check if two tensors are equal
 void allEqual(std::shared_ptr<Tensor> actual, std::shared_ptr<Tensor> expected);
@@ -85,13 +86,14 @@ class Test {
     namespace infiniop_test::name {                                           \
     class Test : public infiniop_test::base::Test {                           \
         double _rtol, _atol;                                                  \
+        bool _equal_nan;                                                      \
                                                                               \
     public:                                                                   \
         static std::string op_name() { return #name; }                        \
         static std::shared_ptr<Test> build(                                   \
             std::unordered_map<std::string, std::vector<uint8_t>> attributes, \
             std::unordered_map<std::string, std::shared_ptr<Tensor>> tensors, \
-            double, double);                                                  \
+            double, double, bool);                                            \
                                                                               \
         static std::vector<std::string> attribute_names();                    \
         static std::vector<std::string> tensor_names();                       \
@@ -109,15 +111,16 @@ class Test {
         struct Attributes;                                                    \
         Attributes *_attributes;                                              \
         Test() = delete;                                                      \
-        Test(double rtol, double atol) : _rtol(rtol), _atol(atol) {}          \
+        Test(double rtol, double atol, bool equal_nan = false)                \
+            : _rtol(rtol), _atol(atol), _equal_nan(equal_nan) {}              \
     };                                                                        \
     }
 
 namespace infiniop_test {
 using BuilderFunc = std::function<std::shared_ptr<infiniop_test::base::Test>(
     std::unordered_map<std::string, std::vector<uint8_t>>,
     std::unordered_map<std::string, std::shared_ptr<Tensor>>,
-    double, double)>;
+    double, double, bool)>;
 
 // Testcase Registry
 // Each testcase should provid a formatted builder, attribute names, and tensor names