<?xml version="1.0" encoding="UTF-8"?>

<!--XBRL Document Created with the Workiva Platform-->
<!--Copyright 2026 Workiva-->
<!--r:019be757-5e31-7922-b287-f8b6d1c74d32,g:88f4fcf6-0468-4702-bc52-ff0c10082c7a-->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:ibm="http://www.ibm.com/20260528" xmlns:xbrli="http://www.xbrl.org/2003/instance" xmlns:dtr-types="http://www.xbrl.org/dtr/type/2024-01-31" attributeFormDefault="unqualified" elementFormDefault="qualified" targetNamespace="http://www.ibm.com/20260528">
  <xs:import namespace="http://fasb.org/us-gaap/2025" schemaLocation="https://xbrl.fasb.org/us-gaap/2025/elts/us-gaap-2025.xsd"/>
  <xs:import namespace="http://www.xbrl.org/2003/instance" schemaLocation="http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd"/>
  <xs:import namespace="http://www.xbrl.org/2003/linkbase" schemaLocation="http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd"/>
  <xs:import namespace="http://www.xbrl.org/dtr/type/2024-01-31" schemaLocation="https://www.xbrl.org/dtr/type/2024-01-31/types.xsd"/>
  <xs:import namespace="http://xbrl.sec.gov/dei/2025" schemaLocation="https://xbrl.sec.gov/dei/2025/dei-2025.xsd"/>
  <xs:import namespace="http://xbrl.sec.gov/exch/2025" schemaLocation="https://xbrl.sec.gov/exch/2025/exch-2025.xsd"/>
  <xs:annotation>
    <xs:appinfo>
      <link:linkbaseRef xmlns:xlink="http://www.w3.org/1999/xlink" xlink:arcrole="http://www.w3.org/1999/xlink/properties/linkbase" xlink:href="ibm-20260528_pre.xml" xlink:role="http://www.xbrl.org/2003/role/presentationLinkbaseRef" xlink:type="simple"/>
      <link:linkbaseRef xmlns:xlink="http://www.w3.org/1999/xlink" xlink:arcrole="http://www.w3.org/1999/xlink/properties/linkbase" xlink:href="ibm-20260528_def.xml" xlink:role="http://www.xbrl.org/2003/role/definitionLinkbaseRef" xlink:type="simple"/>
      <link:linkbaseRef xmlns:xlink="http://www.w3.org/1999/xlink" xlink:arcrole="http://www.w3.org/1999/xlink/properties/linkbase" xlink:href="ibm-20260528_lab.xml" xlink:role="http://www.xbrl.org/2003/role/labelLinkbaseRef" xlink:type="simple"/>
      <link:roleType id="Cover" roleURI="http://www.ibm.com/role/Cover">
        <link:definition>0000001 - Document - Cover</link:definition>
        <link:usedOn>link:presentationLink</link:usedOn>
        <link:usedOn>link:calculationLink</link:usedOn>
        <link:usedOn>link:definitionLink</link:usedOn>
      </link:roleType>
    </xs:appinfo>
  </xs:annotation>
  <xs:element id="ibm_Notes0.300PercentDue2026Member" abstract="true" name="Notes0.300PercentDue2026Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes1.250PercentDue2027Member" abstract="true" name="Notes1.250PercentDue2027Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes3.375PercentDue2027Member" abstract="true" name="Notes3.375PercentDue2027Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes0.300PercentDue2028Member" abstract="true" name="Notes0.300PercentDue2028Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes1.750PercentDue2028Member" abstract="true" name="Notes1.750PercentDue2028Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes1.500PercentDue2029Member" abstract="true" name="Notes1.500PercentDue2029Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes0.875PercentDue2030Member" abstract="true" name="Notes0.875PercentDue2030Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_A2.900NotesDue2030Member" abstract="true" name="A2.900NotesDue2030Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes1.750PercentDue2031Member" abstract="true" name="Notes1.750PercentDue2031Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_A3.000NotesDue2031Member" abstract="true" name="A3.000NotesDue2031Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes3.625PercentDue2031Member" abstract="true" name="Notes3.625PercentDue2031Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes0.650PercentDue2032Member" abstract="true" name="Notes0.650PercentDue2032Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_A3.150NotesDue2033Member" abstract="true" name="A3.150NotesDue2033Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_A3.450NotesDue2034Member" abstract="true" name="A3.450NotesDue2034Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes1.250PercentDue2034Member" abstract="true" name="Notes1.250PercentDue2034Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes3.750PercentDue2035Member" abstract="true" name="Notes3.750PercentDue2035Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_A3.450NotesDue2037Member" abstract="true" name="A3.450NotesDue2037Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_A3.850NotesDue2038Member" abstract="true" name="A3.850NotesDue2038Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes4.875PercentDue2038Member" abstract="true" name="Notes4.875PercentDue2038Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes1.200PercentDue2040Member" abstract="true" name="Notes1.200PercentDue2040Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Notes4.000PercentDue2043Member" abstract="true" name="Notes4.000PercentDue2043Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_A3.800NotesDue2045Member" abstract="true" name="A3.800NotesDue2045Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_FloatingRateNotesDue2028Member" abstract="true" name="FloatingRateNotesDue2028Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Debentures6.22PercentDue2027Member" abstract="true" name="Debentures6.22PercentDue2027Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Debentures6.50PercentDue2028Member" abstract="true" name="Debentures6.50PercentDue2028Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Debentures5.875PercentDue2032Member" abstract="true" name="Debentures5.875PercentDue2032Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Debentures7.00PercentDue2045Member" abstract="true" name="Debentures7.00PercentDue2045Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
  <xs:element id="ibm_Debentures7.125PercentDue2096Member" abstract="true" name="Debentures7.125PercentDue2096Member" nillable="true" xbrli:periodType="duration" substitutionGroup="xbrli:item" type="dtr-types:domainItemType"/>
</xs:schema>
