Material Addition

Users can add images (jpeg, jpg, png, gif), videos (mp4), and audio materials (mp3) through the API. Video formats other than mp4 and audio formats other than mp3 will be automatically converted to mp4 and mp3 formats.

Video Material Addition

Reference data format:

const videoParams = {
  app_id: editor.appid, // Optional, which work the current material belongs to. If not passed, it is a global material
  name: "test.mp4", // Video name
  urls: {
    url: "/uploads/test.mp4", // Video link
    thumb: "/uploads/test_thumb.png", // Cover image
  },
  attrs: {
    size: 1024, // Optional, file size in kb
    type: "video/mp4", // Video type
    duration: 10, // Video duration in seconds
    videoWidth: 1280, // Video width
    videoHeight: 720, // Video height
    noAudioTracks: false, // Optional, whether there are audio tracks
    frames: "/uploads/frames.png", // Optional, record current frame image data. If not passed, the server will handle it automatically
    frameScale: 2, // Optional, frame image scale ratio, default is 2
    wave: "/uploads/wave.json", // Optional, audio wave data in current video
    rotation: false, // Optional, whether the current video has a 90-degree rotation
  },
};

Audio Material Addition

Reference data format:

const videoParams = {
  app_id: editor.appid, // Optional, which work the current material belongs to. If not passed, it is a global material
  name: "test.mp3", // Audio name
  urls: {
    url: "/uploads/test.mp3", // Audio link
  },
  attrs: {
    size: 1024, // Optional, file size in kb
    type: "audio/mp3", // Audio type
    duration: 10, // Audio duration in seconds
    wave: "/uploads/wave.json", // Optional, audio wave data in current audio
  },
};

Image Material Addition

Reference data format:

const videoParams = {
  app_id: editor.appid, // Optional, which work the current material belongs to. If not passed, it is a global material
  name: "test.png", // Image name
  urls: {
    url: "/uploads/test.png", // Image link
    thumb: "/uploads/test_thumb.png", // Cover image
  },
  attrs: {
    size: 1024, // Optional, file size in kb
    type: "image/png", // Image type
    naturalWidth: 1000, // Image width
    naturalHeight: 800, // Image height
  },
};

Note that gif images are different from regular images. Gif images have frame images. Reference parameters:

const videoParams = {
  app_id: editor.appid, // Optional, which work the current material belongs to. If not passed, it is a global material
  name: "test.gif", // Image name
  urls: {
    url: "/uploads/test.gif", // Image link
    thumb: "/uploads/test_thumb.png", // Cover image
  },
  attrs: {
    size: 1024, // Optional, file size in kb
    type: "image/gif", // Image type
    naturalWidth: 1000, // Image width
    naturalHeight: 800, // Image height
    frames: "/uploads/frames.png", // Optional, record current frame image data. If not passed, the server will handle it automatically
  },
};

Parameter Acquisition Instructions

We have encapsulated a JS package to help users obtain parameters for these regular files. You need to introduce the relevant JS package in your project (it needs to be used with /assets/worker and /assets/MediaInfoModule.wasm):

<script src="/assets/medianinfo/index.min.js"></script>

<script>

  // Get parameters for uploaded files
  const info = await window.uploadInfo.getUploadBeforeData(
    v.file.url, // File URL, string
    'video', // Optional, file type: 'audio' | 'image' | 'video' | 'image/svg' | 'image/gif'
    uploadBase64, // Optional, base64 file upload interface for uploading cover images, frame images, audio wave data, etc. If this parameter is not filled in, base64 files will be returned for self-upload
  );
</script>

JS Sample Code

async function main() {
  // Get file extension
  const ext = util.getFileExtension(value);

  // Create a material data
  const obj: any = {
    app_id: editor.appid,
    name: util.getFileNameFromUrl(value),
    urls: {
      url: value,
      thumb: "",
    },
    attrs: {
      type: "video/mp4",
    },
  };
  switch (ext.toLocaleLowerCase()) {
    case "mp4":
      {
        // Resource acquisition
        try {
          const m = (await util.mediaLazy(value)) as any;
          obj.attrs.type = "video/mp4";
          obj.attrs.duration = m.duration;
          obj.attrs.videoWidth = m.videoWidth;
          obj.attrs.videoHeight = m.videoHeight;
          const base64 = await util.drawVideoFrame(m, 130, 1);
          const [res] = await editor.apiServer.uploadBase64({
            content: base64,
            name: util.randomID() + ".png",
          });
          obj.urls.thumb = res.storage_path;
        } catch (err) {
          Toast.error(err);
        }
      }
      break;
    case "jpeg":
    case "png":
    case "jpg":
    case "gif":
      {
        try {
          const m = await util.imgLazy(value);
          obj.attrs.type = "image/" + ext.toLocaleLowerCase();
          obj.attrs.naturalWidth = m.naturalWidth;
          obj.attrs.naturalHeight = m.naturalHeight;
        } catch (err) {
          Toast.error(err);
        }
      }
      break;
    case "mp3": {
      try {
        const m = (await util.mediaLazy(value)) as any;
        obj.attrs.type = "audio/mp3";
        obj.attrs.duration = m.duration;
      } catch (err) {
        Toast.error(err);
      }
    }
  }

  // Add material
  const [res2, err2] = await editor.apiServer.createUserMaterial(obj);
  if (!err2) {
    Toast.success("Add successful");
    // Update the editor's material list. Not required for non-editor pages
    pubsub.publish("addItemToCloudList", res2);
  }
}

/**
 * Get file extension
 */
export function getFileExtension(url: string) {
  url = url.split("?")[0];
  const pathArray = url.split(".");
  if (pathArray.length > 1) {
    return pathArray.pop().toLowerCase();
  } else {
    return null; // No extension
  }
}

/**
 * Get file name
 */
export function getFileNameFromUrl(url) {
  try {
    // Parse URL
    const parsedUrl = new URL(url);
    let path = parsedUrl.pathname;

    // Remove trailing slash from path
    if (path.endsWith("/")) {
      path = path.slice(0, -1);
    }

    // Get the last segment of the path (may contain file name)
    let fileName = path.split("/").pop() || "";

    // If no file name is obtained, check query parameters
    if (!fileName) {
      // Try to find file name-like parameters from query parameters
      const params = parsedUrl.searchParams;
      const fileParams = ["file", "filename", "name", "download"];

      for (const param of fileParams) {
        if (params.has(param)) {
          fileName = params.get(param);
          break;
        }
      }
    }

    // Handle possible URL encoding (such as Chinese)
    fileName = decodeURIComponent(fileName);

    // Remove possible query parameters or anchor residues
    const queryIndex = fileName.indexOf("?");
    if (queryIndex !== -1) {
      fileName = fileName.substring(0, queryIndex);
    }

    const hashIndex = fileName.indexOf("#");
    if (hashIndex !== -1) {
      fileName = fileName.substring(0, hashIndex);
    }

    return fileName || null; // Return null if no file name is found
  } catch (e) {
    console.error("URL parsing failed:", e);
    return null;
  }
}

/**
 * Asynchronously load media resources
 * @param src
 * @param type
 * @returns
 */
export function mediaLazy(
  src: string,
  time?: number,
  type?: "video" | "audio"
): Promise<HTMLVideoElement | HTMLAudioElement> {
  return new Promise((resolve, reject) => {
    const media = document.createElement(type || "video");
    media.preload = "auto";
    media.crossOrigin = "Anonymous";
    media.autoplay = false;
    media.src = src;
    media.setAttribute("playsinline", "");
    media.setAttribute("webkit-playsinline", "");
    media.setAttribute("x5-video-player-type", "h5"); // Additional attribute for WeChat
    media.setAttribute("x5-video-player-fullscreen", "false");
    try {
      media.load();
    } catch (e) {
      console.warn("load() may be restricted in iOS WeChat:", e);
    }

    if (time !== undefined) {
      media.currentTime = time;
    }

    // If in WeChat iOS browser, return the object directly after a 3-second delay because loadedmetadata and canplay cannot be automatically triggered
    if (isIOSWechatBrowser()) {
      setTimeout(() => {
        resolve(media);
      }, 3000);
    } else {
      // Listen to multiple events to ensure compatibility
      const handleLoad = () => {
        console.log("Media loaded successfully");
        cleanup();
        resolve(media);
      };

      const handleError = (err: Event | string) => {
        console.error("Media loading failed", src, err);
        cleanup();
        reject(err);
      };

      const cleanup = () => {
        media.removeEventListener("loadedmetadata", handleLoad);
        media.removeEventListener("canplay", handleLoad);
        media.removeEventListener("error", handleError);
      };
      media.addEventListener("loadedmetadata", handleLoad);
      media.addEventListener("canplay", handleLoad);
      media.addEventListener("error", handleError);
    }
  });
}

/**
 * Get cover image at 1s of video, return base64
 * @param video
 * @param limitWidth
 */
export async function drawVideoFrame(
  video: HTMLVideoElement,
  limitWidth: number,
  time?: number
) {
  video = video.cloneNode() as any;
  video.muted = true;
  video.currentTime = time || 1;
  await video.play();
  video.pause();
  const canvas = document.createElement("canvas"),
    width = video.videoWidth, // Canvas size is the same as the image
    height = video.videoHeight;
  const scale = limitWidth / width;
  canvas.width = limitWidth;
  canvas.height = Math.floor(height * scale);
  canvas.getContext("2d").drawImage(video, 0, 0, canvas.width, canvas.height); // Draw canvas
  const thumb = canvas.toDataURL("image/jpeg");
  return thumb;
}

export const imgLazy = (src: string): Promise<HTMLImageElement> => {
  return new Promise((resolve, reject) => {
    const img = new Image();
    img.crossOrigin = "anonymous";
    img.src = src;
    img.onload = function () {
      resolve(img);
    };
    img.onerror = function () {
      console.error("Image loading failed", src, img);
      reject(img);
    };
  });
};