Update data processing

codegeex/data/process_pretrain_dataset.py

@@ -58,7 +58,7 @@ def process_sample(
     
     try:
         if language is not None and language in LANGUAGE_TAG.keys():
-            code = LANGUAGE_TAG[language] + sample["code"]
+            code = LANGUAGE_TAG[language] + "\n" + sample["code"]
         else:
             code = sample["code"]
     except Exception as e:

codegeex/data/processor.py

@@ -67,6 +67,9 @@ class PromptDatasetProcessor(object):
         """
         Instead of processing lazily, we turn the iterable into a list.
         """
+        if sample is None:
+            return None
+        
         return list(self.process_sample(sample))
 
     def process_sample_(self, sample) -> List[Dict[str, List[int]]]:
@@ -141,6 +144,9 @@ class LabelDatasetProcessor(object):
         """
         Instead of processing lazily, we turn the iterable into a list.
         """
+        if sample is None:
+            return None
+        
         return list(self.process_sample(sample))
 
     def process_sample_(self, sample) -> List[Dict[str, List[int]]]: